{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym, random, pickle, os.path, math, glob\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.autograd as autograd\n",
    "import pdb\n",
    "\n",
    "from atari_wrappers import make_atari, wrap_deepmind,LazyFrames\n",
    "from IPython.display import clear_output\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x28c97d29bc8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPDElEQVR4nO3deawd5XnH8e/j6w3bEC8scmwEJkIsTYShFoFSVSyhoSSFqCUVKKrSCol/khaaqAFaVVHUSiVSlZA/0qgWJEUVZXMgQVYEtRyoVLUymCUEYzvYQMHgYIdiFscLtp/+MWP71r2XO/ee5Z7x+/1I1pnlnDvvePQ775w5c94nMhNJR78pk90ASf1h2KVCGHapEIZdKoRhlwph2KVCdBT2iLgiIjZGxKaIuKVbjZLUfTHR79kjYgj4BXA5sAV4ErguM1/oXvMkdcvUDl57PrApM18CiIh7gauBUcM+PWbkTGaP/ZdnzTw8PSU6aGL3ZN2Ofcd01p6pu6s319jvzUy9VuIx2717B3s/2DniDncS9kXAa8PmtwCf/LAXzGQ2n4zLxvzDcfZvHJreP2f6BJvXXftnDAGw/ZzO2rNg3QcATNu5r+M26cMdPGa/+kRnx2z++vYcsyfXfnfUdZ2EfaR3j//31hcRNwA3AMxkVgebk9SJTsK+BTh52Pxi4I0jn5SZy4HlAMcduygPLDu3g01OngND1XvbzlP2d/R35m6urolO29lxkzSGg8fs/VM7O2YfefnoOGadXI1/Ejg9IpZExHTgWuDh7jRLUrdNuGfPzH0R8WXgUWAI+H5mrutay1rkYw/sHXH5y78/A4ADMwf/wk5pPrZi5GP2ymeqi8P7jznQz+b0RSen8WTmT4CfdKktknrIO+ikQhh2qRCGXSqEYZcKYdilQhh2qRCGXSqEYZcKYdilQnR0B50quxeM8hPKwfgpvkYw2jHLKUfvrc327FIh7Nm74PVLR1tz9PYSbff6JaOtOXqPmT27VAjDLhXC0/iGhvZUv2+e/0xn/2VTd37QjeaogUPH7FmPGdizS8Xoa8++Z94UNl8zGKPFTlxnI5j8z9KhemroQ5+nbirnmO3ZPHr/PWbPHhHfj4htEfH8sGXzI2JVRLxYP87rUlsl9UiT0/h/Bq44YtktwOrMPB1YXc9LGmCNyj9FxKnAysz8eD2/Ebg4M7dGxELg8cw8Y6y/s+ycmfnEoyeP9TRJE3T+p19j7c92j3jv5kQv0J2UmVsB6scTJ9o4Sf3R86vxEXFDRKyNiLXb3+pssH5JEzfRsL9Zn75TP24b7YmZuTwzl2XmshMWDP7VTOloNdGwPwx8sZ7+IvDj7jRHUq80+ertHuC/gDMiYktEXA/cBlweES9S1We/rbfNlNSpMW+qyczrRlk1du1lSQPD22WlQvT1dtkNu+Zy0XN/0M9NSkXZsOuuUdfZs0uF6GvPPmXrELP+/iP93KRUlClbR/96255dKoRhlwph2KVCGHapEIZdKoRhlwph2KVCGHapEIZdKoRhlwph2KVCGHapEIZdKkSTYalOjojHImJ9RKyLiBvr5VaFkVqkSc++D/hqZp4FXAB8KSLOxqowUquMGfbM3JqZT9fT7wHrgUXA1cDBYTHuAj7Xq0ZK6ty4PrPXZaDOBdbQsCrM8CIRez/Y2VlrJU1Y47BHxBzgh8BNmflu09cNLxIxfdrsibRRUhc0CntETKMK+t2Z+WC9uHFVGEmTr8nV+ADuBNZn5reGrbIqjNQiTQacvAj4Y+DnEfFsveyvqKrA3F9XiHkV+HxvmiipG5pUhPkPYMR6z1gVRmoN76CTCmHYpUIYdqkQhl0qhGGXCmHYpUIYdqkQhl0qhGGXCmHYpUIYdqkQhl0qhGGXCmHYpUIYdqkQhl0qhGGXCtFkDLqZEfFERPysrgjzjXr5kohYU1eEuS8ipve+uZImqknPvge4NDPPAZYCV0TEBcA3gW/XFWHeBq7vXTMldapJRZjMzPfr2Wn1vwQuBVbUy60IIw24puPGD9Ujy24DVgGbgR2Zua9+yhaqklAjvdaKMNIAaBT2zNyfmUuBxcD5wFkjPW2U11oRRhoA47oan5k7gMepqrnOjYiDQ1EvBt7obtMkdVOTq/EnRMTcevoY4FNUlVwfA66pn2ZFGGnANakIsxC4KyKGqN4c7s/MlRHxAnBvRPwd8AxViShJA6pJRZjnqMo0H7n8JarP75JawDvopEIYdqkQhl0qhGGXCmHYpUIYdqkQhl0qhGGXCmHYpUIYdqkQhl0qhGGXCmHYpUIYdqkQhl0qhGGXCmHYpUI0Dns9nPQzEbGynrcijNQi4+nZb6QaaPIgK8JILdK0SMRi4DPAHfV8YEUYqVWa9uy3A18DDtTzC7AijNQqTcaN/yywLTOfGr54hKdaEUYaYE3Gjb8IuCoirgRmAsdR9fRzI2Jq3btbEUYacE2quN6amYsz81TgWuCnmfkFrAgjtUon37PfDHwlIjZRfYa3Iow0wJqcxh+SmY9TFXa0IozUMt5BJxXCsEuFMOxSIQy7VAjDLhXCsEuFMOxSIQy7VAjDLhXCsEuFMOxSIQy7VAjDLhXCsEuFGNdPXAfRO0tmArDrhMMjZc15/cCw6T2N/s6O02Yemp69rRpab9r7+0Z7utQ69uxSIVrfsx/s0Xeesv/Qsqm7hg5Nz3l99Nce+7dbDk3fveRHh6Yv+ce/BOCj/2nPrqNHo7BHxCvAe8B+YF9mLouI+cB9wKnAK8AfZebbvWmmpE6N5zT+ksxcmpnL6vlbgNV1RZjV9bykAdXJafzVwMX19F1UY9Pd3GF7Js2F//7lQ9MnvXTgQ54ptVPTnj2Bf4uIpyLihnrZSZm5FaB+PHGkF1oRRhoMTXv2izLzjYg4EVgVERuabiAzlwPLAY47dtGIVWMmy3t/s/jQ9Gn/Z02zr+ukNmnUs2fmG/XjNuAhqiGk34yIhQD147ZeNVJS55rUepsdEccenAZ+F3geeJiqEgxYEUYaeE1O408CHqqqNDMV+NfMfCQingTuj4jrgVeBz/eumZI6NWbY68ov54yw/C3gsl40SlL3ebusVIjW3y4rDbLdC6Yfmt4173DfOuPd6l6OWdv29q0t9uxSIVrfs0+t79OZtuPw+9bQroH6Ol8Fe3/h4R9lvXPm4R9rzXmlWj6rj19Y27NLhTDsUiFafxp//Lpd1cS6yW2HNOjs2aVCGHapEIZdKoRhlwph2KVCGHapEIZdKoRhlwph2KVCGHapEI3CHhFzI2JFRGyIiPURcWFEzI+IVRHxYv04r9eNlTRxTXv27wCPZOaZVENUrceKMFKrjPlDmIg4Dvgd4E8AMnMvsDcijqqKMFIvHP/8rmHTk9gQmvXspwHbgR9ExDMRcUc9pLQVYaQWaRL2qcB5wPcy81xgJ+M4Zc/M5Zm5LDOXTZ82e4LNlNSpJmHfAmzJzDX1/Aqq8FsRRmqRMcOemb8EXouIM+pFlwEvYEUYqVWajlTzZ8DdETEdeAn4U6o3CivCSC3RKOyZ+SywbIRVVoSRWsI76KRCGHapEIZdKoRhlwph2KVCGHapEIZdKoRhlwph2KVCGHapEIZdKoRhlwph2KVCGHapEIZdKoRhlwoxZtgj4oyIeHbYv3cj4iaLREjt0mQMuo2ZuTQzlwK/CfwaeAiLREitMt7T+MuAzZn538DVVMUhqB8/182GSequ8Yb9WuCeerpRkQhJg6Fx2OuRZa8CHhjPBqwIIw2G8fTsvwc8nZlv1vONikRYEUYaDOMJ+3UcPoUHi0RIrdK0Pvss4HLgwWGLbwMuj4gX63W3db95krqlaZGIXwMLjlj2FhaJkFrDO+ikQhh2qRCGXSqEYZcKYdilQhh2qRCGXSqEYZcKYdilQhh2qRCGXSqEYZcK0eiHMN2yZ94UNl8zvZ+blIqyZ/Po/bc9u1SIvvbsn5i3nSf+8J/6uUmpKOcv3z7qOnt2qRCGXSpE02Gp/iIi1kXE8xFxT0TMjIglEbGmrghzXz36rKQB1aT80yLgz4FlmflxYIhq/PhvAt+uK8K8DVzfy4ZK6kzT0/ipwDERMRWYBWwFLgVW1OutCCMNuCa13l4H/gF4lSrk7wBPATsyc1/9tC3Aol41UlLnmpzGz6Oq67YE+Cgwm6pgxJFylNcfqgiz/a39nbRVUgeanMZ/Cng5M7dn5gdUY8f/FjC3Pq0HWAy8MdKLh1eEOWHBUFcaLWn8moT9VeCCiJgVEUE1VvwLwGPANfVzrAgjDbgmn9nXUF2Iexr4ef2a5cDNwFciYhNVAYk7e9hOSR1qWhHm68DXj1j8EnB+11skqSe8g04qhGGXCmHYpUIYdqkQkTnivTC92VjEdmAn8Ku+bbT3jsf9GVRH075As/05JTNPGGlFX8MOEBFrM3NZXzfaQ+7P4Dqa9gU63x9P46VCGHapEJMR9uWTsM1ecn8G19G0L9Dh/vT9M7ukyeFpvFSIvoY9Iq6IiI0RsSkibunntjsVESdHxGMRsb4ej+/Gevn8iFhVj8W3qv79f2tExFBEPBMRK+v51o4tGBFzI2JFRGyoj9OFbT4+3R77sW9hj4gh4LtUA1+cDVwXEWf3a/tdsA/4amaeBVwAfKlu/y3A6nosvtX1fJvcCKwfNt/msQW/AzySmWcC51DtVyuPT0/GfszMvvwDLgQeHTZ/K3Brv7bfg/35MXA5sBFYWC9bCGyc7LaNYx8WUwXgUmAlEFQ3bUwd6ZgN8j/gOOBl6utQw5a38vhQDfP2GjCf6tepK4FPd3J8+nkaf7DxB7V23LqIOBU4F1gDnJSZWwHqxxMnr2XjdjvwNeBAPb+A9o4teBqwHfhB/bHkjoiYTUuPT/Zg7Md+hj1GWNa6rwIiYg7wQ+CmzHx3stszURHxWWBbZj41fPEIT23LMZoKnAd8LzPPpbotuxWn7CPpdOzHkfQz7FuAk4fNjzpu3aCKiGlUQb87Mx+sF78ZEQvr9QuBbZPVvnG6CLgqIl4B7qU6lb+dhmMLDqAtwJasRlaCanSl82jv8elo7MeR9DPsTwKn11cTp1NdbHi4j9vvSD3+3p3A+sz81rBVD1ONwQctGosvM2/NzMWZeSrVsfhpZn6Blo4tmJm/BF6LiDPqRQfHSmzl8aEXYz/2+aLDlcAvgM3AX0/2RZBxtv23qU6ZngOerf9dSfU5dzXwYv04f7LbOoF9uxhYWU+fBjwBbAIeAGZMdvvGsR9LgbX1MfoRMK/Nxwf4BrABeB74F2BGJ8fHO+ikQngHnVQIwy4VwrBLhTDsUiEMu1QIwy4VwrBLhTDsUiH+F2PYU2rydFXiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create and wrap the environment\n",
    "env = make_atari('PongNoFrameskip-v4') # only use in no frameskip environment\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True )\n",
    "n_actions = env.action_space.n\n",
    "state_dim = env.observation_space.shape\n",
    "\n",
    "# env.render()\n",
    "test = env.reset()\n",
    "for i in range(100):\n",
    "    test = env.step(env.action_space.sample())[0]\n",
    "\n",
    "plt.imshow(test._force()[...,0])\n",
    "\n",
    "#plt.imshow(env.render(\"rgb_array\"))\n",
    "# env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class DQN(nn.Module):\n",
    "    def __init__(self, in_channels=4, num_actions=5):\n",
    "        \"\"\"\n",
    "        Initialize a deep Q-learning network as described in\n",
    "        https://storage.googleapis.com/deepmind-data/assets/papers/DeepMindNature14236Paper.pdf\n",
    "        Arguments:\n",
    "            in_channels: number of channel of input.\n",
    "                i.e The number of most recent frames stacked together as describe in the paper\n",
    "            num_actions: number of action-value to output, one-to-one correspondence to action in game.\n",
    "        \"\"\"\n",
    "        super(DQN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)\n",
    "        self.fc4 = nn.Linear(7 * 7 * 64, 512)\n",
    "        self.fc5 = nn.Linear(512, num_actions)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.fc4(x.view(x.size(0), -1)))\n",
    "        return self.fc5(x)\n",
    "\n",
    "class ICM(nn.Module):\n",
    "    def __init__(self, in_channels=4, num_actions=5):\n",
    "        \"\"\"\n",
    "        Initialize a deep Q-learning network as described in\n",
    "        https://storage.googleapis.com/deepmind-data/assets/papers/DeepMindNature14236Paper.pdf\n",
    "        Arguments:\n",
    "            in_channels: number of channel of input.\n",
    "                i.e The number of most recent frames stacked together as describe in the paper\n",
    "            num_actions: number of action-value to output, one-to-one correspondence to action in game.\n",
    "        \"\"\"\n",
    "        super(ICM, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)\n",
    "        self.fc4 = nn.Linear(7 * 7 * 64, 512)\n",
    "                \n",
    "        self.pred_module1 = nn.Linear(512 + num_actions, 256)\n",
    "        self.pred_module2 = nn.Linear(256, 512)\n",
    "            \n",
    "        self.invpred_module1 = nn.Linear(512 + 512, 256)\n",
    "        self.invpred_module2 = nn.Linear(256, num_actions)\n",
    "\n",
    "    def get_feature(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.fc4(x.view(x.size(0), -1)))\n",
    "        return x\n",
    "    \n",
    "    def forward(self, x):\n",
    "        # get feature\n",
    "        feature_x = self.get_feature(x)\n",
    "        return feature_x\n",
    "    \n",
    "    def get_full(self, x, x_next, a_vec):\n",
    "        # get feature\n",
    "        feature_x = self.get_feature(x)\n",
    "        feature_x_next = self.get_feature(x_next)\n",
    "\n",
    "        pred_s_next = self.pred(feature_x, a_vec) # predict next state feature\n",
    "        pred_a_vec = self.invpred(feature_x, feature_x_next) # (inverse) predict action\n",
    "\n",
    "        return pred_s_next, pred_a_vec, feature_x_next\n",
    "\n",
    "    def pred(self, feature_x, a_vec):\n",
    "        # Forward prediction: predict next state feature, given current state feature and action (one-hot)\n",
    "        pred_s_next = F.relu(self.pred_module1( torch.cat([feature_x, a_vec.float()], dim = -1).detach()))\n",
    "        pred_s_next = self.pred_module2(pred_s_next)\n",
    "        return pred_s_next\n",
    "    \n",
    "    def invpred(self,feature_x, feature_x_next):\n",
    "        # Inverse prediction: predict action (one-hot), given current and next state features\n",
    "        pred_a_vec = F.relu(self.invpred_module1(torch.cat([feature_x, feature_x_next], dim = -1)))\n",
    "        pred_a_vec = self.invpred_module2(pred_a_vec)\n",
    "        return F.softmax(pred_a_vec, dim = -1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Memory_Buffer(object):\n",
    "    def __init__(self, memory_size=1000):\n",
    "        self.buffer = []\n",
    "        self.memory_size = memory_size\n",
    "        self.next_idx = 0\n",
    "        \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        data = (state, action, reward, next_state, done)\n",
    "        if len(self.buffer) <= self.memory_size: # buffer not full\n",
    "            self.buffer.append(data)\n",
    "        else: # buffer is full\n",
    "            self.buffer[self.next_idx] = data\n",
    "        self.next_idx = (self.next_idx + 1) % self.memory_size\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        for i in range(batch_size):\n",
    "            idx = random.randint(0, self.size() - 1)\n",
    "            data = self.buffer[idx]\n",
    "            state, action, reward, next_state, done= data\n",
    "            states.append(state)\n",
    "            actions.append(action)\n",
    "            rewards.append(reward)\n",
    "            next_states.append(next_state)\n",
    "            dones.append(done)\n",
    "            \n",
    "            \n",
    "        return np.concatenate(states), actions, rewards, np.concatenate(next_states), dones\n",
    "    \n",
    "    def size(self):\n",
    "        return len(self.buffer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ICM_DQNAgent: \n",
    "    def __init__(self, in_channels = 1, action_space = [], USE_CUDA = False, memory_size = 10000, epsilon  = 1, lr = 1e-4, \n",
    "                 forward_scale = 0.8, inverse_scale = 0.2, Qloss_scale = 0.1, intrinsic_scale= 1, use_extrinsic = True):\n",
    "        self.epsilon = epsilon\n",
    "        self.action_space = action_space\n",
    "        # param for ICM\n",
    "        self.forward_scale = forward_scale # scale for loss function of forward prediction model, 0.8        \n",
    "        self.inverse_scale = inverse_scale # scale for loss function of inverse prediction model, 0.2\n",
    "        self.Qloss_scale = Qloss_scale # scale for loss function of Q value, 1\n",
    "        self.intrinsic_scale = intrinsic_scale # scale for intrinsic reward, 1\n",
    "        self.use_extrinsic = use_extrinsic # whether use extrinsic rewards, if False, only intrinsic reward generated from ICM is used\n",
    "        \n",
    "        self.memory_buffer = Memory_Buffer(memory_size)\n",
    "        self.DQN = DQN(in_channels = in_channels, num_actions = action_space.n)\n",
    "        self.DQN_target = DQN(in_channels = in_channels, num_actions = action_space.n)\n",
    "        self.DQN_target.load_state_dict(self.DQN.state_dict())\n",
    "        self.ICM = ICM(in_channels = in_channels, num_actions = action_space.n)\n",
    "        \n",
    "        self.USE_CUDA = USE_CUDA\n",
    "        if USE_CUDA:\n",
    "            self.DQN = self.DQN.cuda()\n",
    "            self.DQN_target = self.DQN_target.cuda()\n",
    "            self.ICM = self.ICM.cuda()\n",
    "        self.optimizer = optim.Adam(list(self.DQN.parameters())+list(self.ICM.parameters()),lr=lr)\n",
    "\n",
    "    def observe(self, lazyframe):\n",
    "        # from Lazy frame to tensor\n",
    "        state =  torch.from_numpy(lazyframe._force().transpose(2,0,1)[None]/255).float()\n",
    "        if self.USE_CUDA:\n",
    "            state = state.cuda()\n",
    "        return state\n",
    "\n",
    "    def value(self, state):\n",
    "        q_values = self.DQN(state)\n",
    "        return q_values\n",
    "    \n",
    "    def act(self, state, epsilon = None):\n",
    "        \"\"\"\n",
    "        sample actions with epsilon-greedy policy\n",
    "        recap: with p = epsilon pick random action, else pick action with highest Q(s,a)\n",
    "        \"\"\"\n",
    "        if epsilon is None: epsilon = self.epsilon\n",
    "\n",
    "        q_values = self.value(state).cpu().detach().numpy()\n",
    "        if random.random()<epsilon:\n",
    "            aciton = random.randrange(self.action_space.n)\n",
    "        else:\n",
    "            aciton = q_values.argmax(1)[0]\n",
    "        return aciton\n",
    "    \n",
    "    def compute_td_loss(self, states, actions, rewards, next_states, is_done, gamma=0.99):\n",
    "        \"\"\" Compute td loss using torch operations only. Use the formula above. \"\"\"\n",
    "        actions = torch.tensor(actions).long()    # shape: [batch_size]\n",
    "        rewards = torch.tensor(rewards, dtype =torch.float)  # shape: [batch_size]\n",
    "        is_done = torch.tensor(is_done).type(torch.bool)  # shape: [batch_size]\n",
    "        \n",
    "        if self.USE_CUDA:\n",
    "            actions = actions.cuda()\n",
    "            rewards = rewards.cuda()\n",
    "            is_done = is_done.cuda()\n",
    "\n",
    "        # get q-values for all actions in current states\n",
    "        predicted_qvalues = self.DQN(states)\n",
    "        \n",
    "        # get ICM results\n",
    "        a_vec = F.one_hot(actions, num_classes = self.action_space.n) # convert action from int to one-hot format\n",
    "        pred_s_next, pred_a_vec, feature_x_next = self.ICM.get_full(states, next_states, a_vec)\n",
    "        # calculate forward prediction and inverse prediction loss\n",
    "        forward_loss = F.mse_loss(pred_s_next, feature_x_next.detach(), reduction='none')\n",
    "        inverse_pred_loss = F.cross_entropy(pred_a_vec, actions.detach(), reduction='none')\n",
    "        \n",
    "        # calculate rewards\n",
    "        intrinsic_rewards = self.intrinsic_scale * forward_loss.mean(-1)\n",
    "        total_rewards = intrinsic_rewards.clone()\n",
    "        if self.use_extrinsic:\n",
    "            total_rewards += rewards\n",
    "            \n",
    "        # select q-values for chosen actions\n",
    "        predicted_qvalues_for_actions = predicted_qvalues[\n",
    "          range(states.shape[0]), actions\n",
    "        ]\n",
    "\n",
    "        # compute q-values for all actions in next states\n",
    "        predicted_next_qvalues = self.DQN_target(next_states) # YOUR CODE\n",
    "\n",
    "        # compute V*(next_states) using predicted next q-values\n",
    "        next_state_values =  predicted_next_qvalues.max(-1)[0] # YOUR CODE\n",
    "        \n",
    "        # compute \"target q-values\" for loss - it's what's inside square parentheses in the above formula.\n",
    "        target_qvalues_for_actions = total_rewards + gamma *next_state_values # YOUR CODE\n",
    "\n",
    "        # at the last state we shall use simplified formula: Q(s,a) = r(s,a) since s' doesn't exist\n",
    "        target_qvalues_for_actions = torch.where(\n",
    "            is_done, total_rewards, target_qvalues_for_actions)\n",
    "\n",
    "        # mean squared error loss to minimize\n",
    "        #loss = torch.mean((predicted_qvalues_for_actions -\n",
    "        #                   target_qvalues_for_actions.detach()) ** 2)\n",
    "        Q_loss = F.smooth_l1_loss(predicted_qvalues_for_actions, target_qvalues_for_actions.detach())\n",
    "        loss = self.Qloss_scale*Q_loss + self.forward_scale*forward_loss.mean() + self.inverse_scale* inverse_pred_loss.mean()\n",
    "\n",
    "        return loss, Q_loss.item(), forward_loss.mean().item(), inverse_pred_loss.mean().item(), intrinsic_rewards.mean().item()\n",
    "    \n",
    "    def sample_from_buffer(self, batch_size):\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        for i in range(batch_size):\n",
    "            idx = random.randint(0, self.memory_buffer.size() - 1)\n",
    "            data = self.memory_buffer.buffer[idx]\n",
    "            frame, action, reward, next_frame, done= data\n",
    "            states.append(self.observe(frame))\n",
    "            actions.append(action)\n",
    "            rewards.append(reward)\n",
    "            next_states.append(self.observe(next_frame))\n",
    "            dones.append(done)\n",
    "        return torch.cat(states), actions, rewards, torch.cat(next_states), dones\n",
    "\n",
    "    def learn_from_experience(self, batch_size):\n",
    "        if self.memory_buffer.size() > batch_size:\n",
    "            states, actions, rewards, next_states, dones = self.sample_from_buffer(batch_size)\n",
    "            td_loss, Q_loss, forward_loss, inverse_pred_loss,intrinsic_rewards = self.compute_td_loss(states, actions, rewards, next_states, dones)\n",
    "            self.optimizer.zero_grad()\n",
    "            td_loss.backward()\n",
    "            for param in list(self.DQN.parameters())+list(self.ICM.parameters()):\n",
    "                param.grad.data.clamp_(-1, 1)\n",
    "\n",
    "            self.optimizer.step()\n",
    "            return(td_loss.item(),Q_loss, forward_loss, inverse_pred_loss,intrinsic_rewards)\n",
    "        else:\n",
    "            return(0,0,0,0,0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:NaN or Inf found in input tensor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames:     0, reward:   nan, total_loss: 0.000000, forward_loss: 0.000000, inverse_pred_loss: 0.000000, Q_loss: 0.000000, intrinsic_rewards: 0.000000, epsilon: 1.000000, episode:    0\n",
      "frames:  1000, reward: -21.000000, total_loss: 0.000000, forward_loss: 0.000000, inverse_pred_loss: 0.000000, Q_loss: 0.000000, intrinsic_rewards: 0.000000, epsilon: 0.980397, episode:    1\n",
      "frames:  2000, reward: -21.000000, total_loss: 0.000000, forward_loss: 0.000000, inverse_pred_loss: 0.000000, Q_loss: 0.000000, intrinsic_rewards: 0.000000, epsilon: 0.961182, episode:    2\n",
      "frames:  3000, reward: -20.333333, total_loss: 0.000000, forward_loss: 0.000000, inverse_pred_loss: 0.000000, Q_loss: 0.000000, intrinsic_rewards: 0.000000, epsilon: 0.942347, episode:    3\n",
      "frames:  4000, reward: -20.000000, total_loss: 0.000000, forward_loss: 0.000000, inverse_pred_loss: 0.000000, Q_loss: 0.000000, intrinsic_rewards: 0.000000, epsilon: 0.923885, episode:    4\n",
      "frames:  5000, reward: -20.000000, total_loss: 1.818037, forward_loss: 0.001793, inverse_pred_loss: 1.791011, Q_loss: 0.025232, intrinsic_rewards: 0.179305, epsilon: 0.905789, episode:    5\n",
      "frames:  6000, reward: -20.166667, total_loss: 1.778422, forward_loss: 0.000015, inverse_pred_loss: 1.778036, Q_loss: 0.000370, intrinsic_rewards: 0.001548, epsilon: 0.888051, episode:    6\n",
      "frames:  7000, reward: -20.285714, total_loss: 1.797396, forward_loss: 0.000009, inverse_pred_loss: 1.797019, Q_loss: 0.000368, intrinsic_rewards: 0.000932, epsilon: 0.870665, episode:    7\n",
      "frames:  8000, reward: -20.375000, total_loss: 1.785739, forward_loss: 0.000005, inverse_pred_loss: 1.785331, Q_loss: 0.000403, intrinsic_rewards: 0.000456, epsilon: 0.853622, episode:    8\n",
      "frames:  9000, reward: -20.333333, total_loss: 1.802485, forward_loss: 0.000006, inverse_pred_loss: 1.787580, Q_loss: 0.014898, intrinsic_rewards: 0.000616, epsilon: 0.836918, episode:    9\n",
      "frames: 10000, reward: -20.400000, total_loss: 1.801928, forward_loss: 0.000003, inverse_pred_loss: 1.786722, Q_loss: 0.015203, intrinsic_rewards: 0.000271, epsilon: 0.820543, episode:   10\n",
      "frames: 11000, reward: -20.300000, total_loss: 1.817238, forward_loss: 0.000002, inverse_pred_loss: 1.817010, Q_loss: 0.000226, intrinsic_rewards: 0.000169, epsilon: 0.804494, episode:   11\n",
      "frames: 12000, reward: -20.200000, total_loss: 1.815264, forward_loss: 0.000001, inverse_pred_loss: 1.785020, Q_loss: 0.030243, intrinsic_rewards: 0.000063, epsilon: 0.788762, episode:   12\n",
      "frames: 13000, reward: -20.100000, total_loss: 1.804464, forward_loss: 0.000002, inverse_pred_loss: 1.789254, Q_loss: 0.015208, intrinsic_rewards: 0.000150, epsilon: 0.773341, episode:   13\n",
      "frames: 14000, reward: -20.200000, total_loss: 1.817084, forward_loss: 0.000000, inverse_pred_loss: 1.800987, Q_loss: 0.016097, intrinsic_rewards: 0.000017, epsilon: 0.758226, episode:   14\n",
      "frames: 15000, reward: -20.200000, total_loss: 1.790265, forward_loss: 0.000000, inverse_pred_loss: 1.784170, Q_loss: 0.006095, intrinsic_rewards: 0.000048, epsilon: 0.743410, episode:   15\n",
      "frames: 16000, reward: -20.000000, total_loss: 1.817883, forward_loss: 0.000000, inverse_pred_loss: 1.795752, Q_loss: 0.022131, intrinsic_rewards: 0.000033, epsilon: 0.728888, episode:   16\n",
      "frames: 17000, reward: -19.900000, total_loss: 1.786215, forward_loss: 0.000000, inverse_pred_loss: 1.785389, Q_loss: 0.000826, intrinsic_rewards: 0.000047, epsilon: 0.714653, episode:   17\n",
      "frames: 18000, reward: -19.800000, total_loss: 1.778926, forward_loss: 0.000002, inverse_pred_loss: 1.758753, Q_loss: 0.020171, intrinsic_rewards: 0.000230, epsilon: 0.700700, episode:   19\n",
      "frames: 19000, reward: -19.700000, total_loss: 1.792477, forward_loss: 0.000022, inverse_pred_loss: 1.791174, Q_loss: 0.001281, intrinsic_rewards: 0.002226, epsilon: 0.687023, episode:   20\n",
      "frames: 20000, reward: -19.800000, total_loss: 1.788405, forward_loss: 0.000001, inverse_pred_loss: 1.787944, Q_loss: 0.000460, intrinsic_rewards: 0.000070, epsilon: 0.673617, episode:   21\n",
      "frames: 21000, reward: -19.900000, total_loss: 1.777148, forward_loss: 0.000000, inverse_pred_loss: 1.775415, Q_loss: 0.001734, intrinsic_rewards: 0.000017, epsilon: 0.660476, episode:   22\n",
      "frames: 22000, reward: -20.200000, total_loss: 1.791770, forward_loss: 0.000001, inverse_pred_loss: 1.790226, Q_loss: 0.001543, intrinsic_rewards: 0.000097, epsilon: 0.647596, episode:   23\n",
      "frames: 23000, reward: -20.200000, total_loss: 1.785312, forward_loss: 0.000000, inverse_pred_loss: 1.782972, Q_loss: 0.002339, intrinsic_rewards: 0.000037, epsilon: 0.634971, episode:   24\n",
      "frames: 24000, reward: -20.300000, total_loss: 1.799213, forward_loss: 0.000003, inverse_pred_loss: 1.795537, Q_loss: 0.003673, intrinsic_rewards: 0.000338, epsilon: 0.622596, episode:   25\n",
      "frames: 25000, reward: -20.400000, total_loss: 1.801813, forward_loss: 0.000001, inverse_pred_loss: 1.799142, Q_loss: 0.002670, intrinsic_rewards: 0.000075, epsilon: 0.610465, episode:   26\n",
      "frames: 26000, reward: -20.400000, total_loss: 1.793205, forward_loss: 0.000000, inverse_pred_loss: 1.792382, Q_loss: 0.000823, intrinsic_rewards: 0.000023, epsilon: 0.598575, episode:   27\n",
      "frames: 27000, reward: -20.500000, total_loss: 1.783151, forward_loss: 0.000002, inverse_pred_loss: 1.782499, Q_loss: 0.000651, intrinsic_rewards: 0.000153, epsilon: 0.586921, episode:   28\n",
      "frames: 28000, reward: -20.700000, total_loss: 1.789459, forward_loss: 0.000000, inverse_pred_loss: 1.789020, Q_loss: 0.000439, intrinsic_rewards: 0.000017, epsilon: 0.575497, episode:   30\n",
      "frames: 29000, reward: -20.600000, total_loss: 1.788540, forward_loss: 0.000000, inverse_pred_loss: 1.785924, Q_loss: 0.002616, intrinsic_rewards: 0.000043, epsilon: 0.564299, episode:   31\n",
      "frames: 30000, reward: -20.500000, total_loss: 1.776574, forward_loss: 0.000000, inverse_pred_loss: 1.776074, Q_loss: 0.000500, intrinsic_rewards: 0.000007, epsilon: 0.553324, episode:   32\n",
      "frames: 31000, reward: -20.400000, total_loss: 1.790704, forward_loss: 0.000000, inverse_pred_loss: 1.790322, Q_loss: 0.000383, intrinsic_rewards: 0.000008, epsilon: 0.542565, episode:   33\n",
      "frames: 32000, reward: -20.500000, total_loss: 1.793847, forward_loss: 0.000000, inverse_pred_loss: 1.792469, Q_loss: 0.001378, intrinsic_rewards: 0.000009, epsilon: 0.532019, episode:   34\n",
      "frames: 33000, reward: -20.400000, total_loss: 1.786222, forward_loss: 0.000000, inverse_pred_loss: 1.785575, Q_loss: 0.000647, intrinsic_rewards: 0.000028, epsilon: 0.521683, episode:   35\n",
      "frames: 34000, reward: -20.300000, total_loss: 1.790720, forward_loss: 0.000000, inverse_pred_loss: 1.788788, Q_loss: 0.001932, intrinsic_rewards: 0.000041, epsilon: 0.511551, episode:   36\n",
      "frames: 35000, reward: -20.100000, total_loss: 1.790795, forward_loss: 0.000000, inverse_pred_loss: 1.787391, Q_loss: 0.003404, intrinsic_rewards: 0.000015, epsilon: 0.501619, episode:   37\n",
      "frames: 36000, reward: -19.900000, total_loss: 1.780023, forward_loss: 0.000000, inverse_pred_loss: 1.775528, Q_loss: 0.004495, intrinsic_rewards: 0.000011, epsilon: 0.491885, episode:   38\n",
      "frames: 37000, reward: -19.900000, total_loss: 1.799471, forward_loss: 0.000000, inverse_pred_loss: 1.795351, Q_loss: 0.004121, intrinsic_rewards: 0.000011, epsilon: 0.482343, episode:   39\n",
      "frames: 38000, reward: -19.600000, total_loss: 1.795350, forward_loss: 0.000000, inverse_pred_loss: 1.791945, Q_loss: 0.003404, intrinsic_rewards: 0.000027, epsilon: 0.472990, episode:   40\n",
      "frames: 39000, reward: -19.700000, total_loss: 1.812171, forward_loss: 0.000000, inverse_pred_loss: 1.808317, Q_loss: 0.003855, intrinsic_rewards: 0.000007, epsilon: 0.463822, episode:   41\n",
      "frames: 40000, reward: -19.500000, total_loss: 1.805751, forward_loss: 0.000000, inverse_pred_loss: 1.804984, Q_loss: 0.000767, intrinsic_rewards: 0.000029, epsilon: 0.454836, episode:   42\n",
      "frames: 41000, reward: -19.600000, total_loss: 1.799158, forward_loss: 0.000000, inverse_pred_loss: 1.792982, Q_loss: 0.006176, intrinsic_rewards: 0.000032, epsilon: 0.446027, episode:   43\n",
      "frames: 42000, reward: -19.500000, total_loss: 1.789437, forward_loss: 0.000000, inverse_pred_loss: 1.784514, Q_loss: 0.004923, intrinsic_rewards: 0.000008, epsilon: 0.437393, episode:   44\n",
      "frames: 43000, reward: -19.300000, total_loss: 1.798639, forward_loss: 0.000000, inverse_pred_loss: 1.794886, Q_loss: 0.003753, intrinsic_rewards: 0.000008, epsilon: 0.428930, episode:   45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 44000, reward: -19.500000, total_loss: 1.778721, forward_loss: 0.000000, inverse_pred_loss: 1.776430, Q_loss: 0.002292, intrinsic_rewards: 0.000006, epsilon: 0.420635, episode:   46\n",
      "frames: 45000, reward: -19.600000, total_loss: 1.787947, forward_loss: 0.000000, inverse_pred_loss: 1.786723, Q_loss: 0.001223, intrinsic_rewards: 0.000041, epsilon: 0.412504, episode:   47\n",
      "frames: 46000, reward: -19.800000, total_loss: 1.792390, forward_loss: 0.000000, inverse_pred_loss: 1.788144, Q_loss: 0.004245, intrinsic_rewards: 0.000023, epsilon: 0.404534, episode:   48\n",
      "frames: 47000, reward: -19.700000, total_loss: 1.766752, forward_loss: 0.000000, inverse_pred_loss: 1.764675, Q_loss: 0.002076, intrinsic_rewards: 0.000011, epsilon: 0.396722, episode:   49\n",
      "frames: 48000, reward: -20.000000, total_loss: 1.800201, forward_loss: 0.000000, inverse_pred_loss: 1.791832, Q_loss: 0.008369, intrinsic_rewards: 0.000015, epsilon: 0.389064, episode:   50\n",
      "frames: 49000, reward: -20.000000, total_loss: 1.784478, forward_loss: 0.000000, inverse_pred_loss: 1.782134, Q_loss: 0.002343, intrinsic_rewards: 0.000040, epsilon: 0.381558, episode:   51\n",
      "frames: 50000, reward: -20.300000, total_loss: 1.804910, forward_loss: 0.000000, inverse_pred_loss: 1.802979, Q_loss: 0.001931, intrinsic_rewards: 0.000009, epsilon: 0.374201, episode:   52\n",
      "frames: 51000, reward: -20.300000, total_loss: 1.784726, forward_loss: 0.000000, inverse_pred_loss: 1.782913, Q_loss: 0.001813, intrinsic_rewards: 0.000035, epsilon: 0.366989, episode:   52\n",
      "frames: 52000, reward: -19.900000, total_loss: 1.783526, forward_loss: 0.000000, inverse_pred_loss: 1.781258, Q_loss: 0.002267, intrinsic_rewards: 0.000036, epsilon: 0.359920, episode:   53\n",
      "frames: 53000, reward: -19.800000, total_loss: 1.796530, forward_loss: 0.000000, inverse_pred_loss: 1.794631, Q_loss: 0.001899, intrinsic_rewards: 0.000026, epsilon: 0.352991, episode:   54\n",
      "frames: 54000, reward: -19.900000, total_loss: 1.785335, forward_loss: 0.000000, inverse_pred_loss: 1.778856, Q_loss: 0.006478, intrinsic_rewards: 0.000007, epsilon: 0.346200, episode:   55\n",
      "frames: 55000, reward: -19.900000, total_loss: 1.791065, forward_loss: 0.000000, inverse_pred_loss: 1.788134, Q_loss: 0.002932, intrinsic_rewards: 0.000005, epsilon: 0.339542, episode:   55\n",
      "frames: 56000, reward: -19.500000, total_loss: 1.793824, forward_loss: 0.000000, inverse_pred_loss: 1.790836, Q_loss: 0.002987, intrinsic_rewards: 0.000040, epsilon: 0.333017, episode:   56\n",
      "frames: 57000, reward: -19.600000, total_loss: 1.834683, forward_loss: 0.000001, inverse_pred_loss: 1.829816, Q_loss: 0.004865, intrinsic_rewards: 0.000075, epsilon: 0.326621, episode:   57\n",
      "frames: 58000, reward: -19.600000, total_loss: 1.795145, forward_loss: 0.000000, inverse_pred_loss: 1.793213, Q_loss: 0.001932, intrinsic_rewards: 0.000019, epsilon: 0.320351, episode:   57\n",
      "frames: 59000, reward: -19.200000, total_loss: 1.774604, forward_loss: 0.000001, inverse_pred_loss: 1.772996, Q_loss: 0.001607, intrinsic_rewards: 0.000053, epsilon: 0.314206, episode:   58\n",
      "frames: 60000, reward: -19.200000, total_loss: 1.802976, forward_loss: 0.000000, inverse_pred_loss: 1.797323, Q_loss: 0.005653, intrinsic_rewards: 0.000050, epsilon: 0.308182, episode:   59\n",
      "frames: 61000, reward: -19.100000, total_loss: 1.777189, forward_loss: 0.000000, inverse_pred_loss: 1.773777, Q_loss: 0.003412, intrinsic_rewards: 0.000045, epsilon: 0.302278, episode:   60\n",
      "frames: 62000, reward: -19.100000, total_loss: 1.793349, forward_loss: 0.000000, inverse_pred_loss: 1.789264, Q_loss: 0.004085, intrinsic_rewards: 0.000014, epsilon: 0.296490, episode:   60\n",
      "frames: 63000, reward: -18.700000, total_loss: 1.772385, forward_loss: 0.000000, inverse_pred_loss: 1.769524, Q_loss: 0.002861, intrinsic_rewards: 0.000028, epsilon: 0.290817, episode:   61\n",
      "frames: 64000, reward: -18.700000, total_loss: 1.805503, forward_loss: 0.000001, inverse_pred_loss: 1.802877, Q_loss: 0.002626, intrinsic_rewards: 0.000072, epsilon: 0.285257, episode:   61\n",
      "frames: 65000, reward: -18.200000, total_loss: 1.806413, forward_loss: 0.000000, inverse_pred_loss: 1.803396, Q_loss: 0.003017, intrinsic_rewards: 0.000021, epsilon: 0.279806, episode:   62\n",
      "frames: 66000, reward: -18.600000, total_loss: 1.767735, forward_loss: 0.000000, inverse_pred_loss: 1.759091, Q_loss: 0.008643, intrinsic_rewards: 0.000009, epsilon: 0.274464, episode:   63\n",
      "frames: 67000, reward: -18.600000, total_loss: 1.775160, forward_loss: 0.000000, inverse_pred_loss: 1.772626, Q_loss: 0.002534, intrinsic_rewards: 0.000007, epsilon: 0.269227, episode:   63\n",
      "frames: 68000, reward: -18.300000, total_loss: 1.768153, forward_loss: 0.000000, inverse_pred_loss: 1.766822, Q_loss: 0.001331, intrinsic_rewards: 0.000018, epsilon: 0.264094, episode:   64\n",
      "frames: 69000, reward: -18.300000, total_loss: 1.786343, forward_loss: 0.000000, inverse_pred_loss: 1.782079, Q_loss: 0.004265, intrinsic_rewards: 0.000026, epsilon: 0.259063, episode:   65\n",
      "frames: 70000, reward: -18.300000, total_loss: 1.799023, forward_loss: 0.000000, inverse_pred_loss: 1.796518, Q_loss: 0.002505, intrinsic_rewards: 0.000021, epsilon: 0.254131, episode:   65\n",
      "frames: 71000, reward: -18.400000, total_loss: 1.777105, forward_loss: 0.000000, inverse_pred_loss: 1.775286, Q_loss: 0.001819, intrinsic_rewards: 0.000014, epsilon: 0.249297, episode:   66\n",
      "frames: 72000, reward: -18.400000, total_loss: 1.776726, forward_loss: 0.000000, inverse_pred_loss: 1.773096, Q_loss: 0.003630, intrinsic_rewards: 0.000030, epsilon: 0.244558, episode:   67\n",
      "frames: 73000, reward: -18.600000, total_loss: 1.755039, forward_loss: 0.000000, inverse_pred_loss: 1.752268, Q_loss: 0.002771, intrinsic_rewards: 0.000018, epsilon: 0.239914, episode:   68\n",
      "frames: 74000, reward: -18.600000, total_loss: 1.743769, forward_loss: 0.000000, inverse_pred_loss: 1.742205, Q_loss: 0.001563, intrinsic_rewards: 0.000004, epsilon: 0.235361, episode:   68\n",
      "frames: 75000, reward: -18.200000, total_loss: 1.814796, forward_loss: 0.000000, inverse_pred_loss: 1.813562, Q_loss: 0.001234, intrinsic_rewards: 0.000029, epsilon: 0.230899, episode:   69\n",
      "frames: 76000, reward: -17.900000, total_loss: 1.762005, forward_loss: 0.000001, inverse_pred_loss: 1.760608, Q_loss: 0.001397, intrinsic_rewards: 0.000051, epsilon: 0.226525, episode:   70\n",
      "frames: 77000, reward: -17.900000, total_loss: 1.751726, forward_loss: 0.000000, inverse_pred_loss: 1.750428, Q_loss: 0.001298, intrinsic_rewards: 0.000010, epsilon: 0.222237, episode:   70\n",
      "frames: 78000, reward: -17.800000, total_loss: 1.781671, forward_loss: 0.000000, inverse_pred_loss: 1.778002, Q_loss: 0.003669, intrinsic_rewards: 0.000012, epsilon: 0.218035, episode:   71\n",
      "frames: 79000, reward: -17.800000, total_loss: 1.776001, forward_loss: 0.000000, inverse_pred_loss: 1.769519, Q_loss: 0.006482, intrinsic_rewards: 0.000020, epsilon: 0.213915, episode:   71\n",
      "frames: 80000, reward: -17.500000, total_loss: 1.799103, forward_loss: 0.000000, inverse_pred_loss: 1.797237, Q_loss: 0.001866, intrinsic_rewards: 0.000042, epsilon: 0.209878, episode:   72\n",
      "frames: 81000, reward: -17.500000, total_loss: 1.802907, forward_loss: 0.000000, inverse_pred_loss: 1.800321, Q_loss: 0.002586, intrinsic_rewards: 0.000018, epsilon: 0.205920, episode:   72\n",
      "frames: 82000, reward: -17.200000, total_loss: 1.803063, forward_loss: 0.000000, inverse_pred_loss: 1.799649, Q_loss: 0.003414, intrinsic_rewards: 0.000003, epsilon: 0.202040, episode:   73\n",
      "frames: 83000, reward: -17.200000, total_loss: 1.776606, forward_loss: 0.000000, inverse_pred_loss: 1.774424, Q_loss: 0.002181, intrinsic_rewards: 0.000041, epsilon: 0.198238, episode:   73\n",
      "frames: 84000, reward: -17.400000, total_loss: 1.774423, forward_loss: 0.000000, inverse_pred_loss: 1.772778, Q_loss: 0.001644, intrinsic_rewards: 0.000014, epsilon: 0.194510, episode:   74\n",
      "frames: 85000, reward: -17.500000, total_loss: 1.790236, forward_loss: 0.000002, inverse_pred_loss: 1.788445, Q_loss: 0.001789, intrinsic_rewards: 0.000194, epsilon: 0.190857, episode:   75\n",
      "frames: 86000, reward: -17.500000, total_loss: 1.796911, forward_loss: 0.000000, inverse_pred_loss: 1.794122, Q_loss: 0.002788, intrinsic_rewards: 0.000049, epsilon: 0.187275, episode:   75\n",
      "frames: 87000, reward: -17.500000, total_loss: 1.785336, forward_loss: 0.000000, inverse_pred_loss: 1.782911, Q_loss: 0.002425, intrinsic_rewards: 0.000040, epsilon: 0.183765, episode:   75\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 88000, reward: -17.200000, total_loss: 1.772214, forward_loss: 0.000001, inverse_pred_loss: 1.767942, Q_loss: 0.004271, intrinsic_rewards: 0.000052, epsilon: 0.180324, episode:   76\n",
      "frames: 89000, reward: -17.100000, total_loss: 1.767155, forward_loss: 0.000000, inverse_pred_loss: 1.765419, Q_loss: 0.001737, intrinsic_rewards: 0.000013, epsilon: 0.176952, episode:   77\n",
      "frames: 90000, reward: -17.100000, total_loss: 1.777267, forward_loss: 0.000000, inverse_pred_loss: 1.774771, Q_loss: 0.002496, intrinsic_rewards: 0.000005, epsilon: 0.173646, episode:   77\n",
      "frames: 91000, reward: -17.100000, total_loss: 1.759823, forward_loss: 0.000000, inverse_pred_loss: 1.758573, Q_loss: 0.001250, intrinsic_rewards: 0.000003, epsilon: 0.170405, episode:   78\n",
      "frames: 92000, reward: -17.100000, total_loss: 1.761452, forward_loss: 0.000000, inverse_pred_loss: 1.759375, Q_loss: 0.002077, intrinsic_rewards: 0.000015, epsilon: 0.167229, episode:   78\n",
      "frames: 93000, reward: -17.300000, total_loss: 1.842617, forward_loss: 0.000000, inverse_pred_loss: 1.835240, Q_loss: 0.007377, intrinsic_rewards: 0.000006, epsilon: 0.164116, episode:   79\n",
      "frames: 94000, reward: -17.300000, total_loss: 1.791921, forward_loss: 0.000000, inverse_pred_loss: 1.782800, Q_loss: 0.009121, intrinsic_rewards: 0.000007, epsilon: 0.161064, episode:   79\n",
      "frames: 95000, reward: -17.600000, total_loss: 1.787270, forward_loss: 0.000000, inverse_pred_loss: 1.785756, Q_loss: 0.001514, intrinsic_rewards: 0.000004, epsilon: 0.158073, episode:   80\n",
      "frames: 96000, reward: -17.600000, total_loss: 1.774849, forward_loss: 0.000000, inverse_pred_loss: 1.774000, Q_loss: 0.000849, intrinsic_rewards: 0.000013, epsilon: 0.155141, episode:   80\n",
      "frames: 97000, reward: -17.900000, total_loss: 1.783873, forward_loss: 0.000000, inverse_pred_loss: 1.781899, Q_loss: 0.001974, intrinsic_rewards: 0.000007, epsilon: 0.152267, episode:   81\n",
      "frames: 98000, reward: -17.900000, total_loss: 1.765245, forward_loss: 0.000000, inverse_pred_loss: 1.764124, Q_loss: 0.001121, intrinsic_rewards: 0.000004, epsilon: 0.149450, episode:   81\n",
      "frames: 99000, reward: -18.200000, total_loss: 1.776521, forward_loss: 0.000000, inverse_pred_loss: 1.774788, Q_loss: 0.001733, intrinsic_rewards: 0.000005, epsilon: 0.146689, episode:   82\n",
      "frames: 100000, reward: -18.400000, total_loss: 1.740452, forward_loss: 0.000000, inverse_pred_loss: 1.739091, Q_loss: 0.001361, intrinsic_rewards: 0.000007, epsilon: 0.143982, episode:   83\n",
      "frames: 101000, reward: -18.400000, total_loss: 1.780714, forward_loss: 0.000000, inverse_pred_loss: 1.778529, Q_loss: 0.002184, intrinsic_rewards: 0.000023, epsilon: 0.141329, episode:   83\n",
      "frames: 102000, reward: -18.400000, total_loss: 1.759175, forward_loss: 0.000000, inverse_pred_loss: 1.756659, Q_loss: 0.002517, intrinsic_rewards: 0.000003, epsilon: 0.138728, episode:   83\n",
      "frames: 103000, reward: -18.000000, total_loss: 1.750069, forward_loss: 0.000000, inverse_pred_loss: 1.748598, Q_loss: 0.001470, intrinsic_rewards: 0.000014, epsilon: 0.136179, episode:   84\n",
      "frames: 104000, reward: -18.000000, total_loss: 1.754120, forward_loss: 0.000000, inverse_pred_loss: 1.752268, Q_loss: 0.001852, intrinsic_rewards: 0.000010, epsilon: 0.133681, episode:   85\n",
      "frames: 105000, reward: -18.000000, total_loss: 1.835310, forward_loss: 0.000000, inverse_pred_loss: 1.833477, Q_loss: 0.001833, intrinsic_rewards: 0.000001, epsilon: 0.131232, episode:   85\n",
      "frames: 106000, reward: -18.000000, total_loss: 1.777726, forward_loss: 0.000000, inverse_pred_loss: 1.776770, Q_loss: 0.000957, intrinsic_rewards: 0.000002, epsilon: 0.128831, episode:   85\n",
      "frames: 107000, reward: -18.000000, total_loss: 1.826308, forward_loss: 0.000000, inverse_pred_loss: 1.824956, Q_loss: 0.001351, intrinsic_rewards: 0.000003, epsilon: 0.126478, episode:   86\n",
      "frames: 108000, reward: -18.000000, total_loss: 1.762350, forward_loss: 0.000000, inverse_pred_loss: 1.761241, Q_loss: 0.001108, intrinsic_rewards: 0.000032, epsilon: 0.124172, episode:   86\n",
      "frames: 109000, reward: -17.800000, total_loss: 1.792774, forward_loss: 0.000000, inverse_pred_loss: 1.791859, Q_loss: 0.000915, intrinsic_rewards: 0.000009, epsilon: 0.121911, episode:   87\n",
      "frames: 110000, reward: -17.800000, total_loss: 1.816188, forward_loss: 0.000000, inverse_pred_loss: 1.813623, Q_loss: 0.002565, intrinsic_rewards: 0.000021, epsilon: 0.119695, episode:   87\n",
      "frames: 111000, reward: -17.400000, total_loss: 1.756139, forward_loss: 0.000001, inverse_pred_loss: 1.754597, Q_loss: 0.001541, intrinsic_rewards: 0.000057, epsilon: 0.117523, episode:   88\n",
      "frames: 112000, reward: -17.400000, total_loss: 1.761915, forward_loss: 0.000000, inverse_pred_loss: 1.760884, Q_loss: 0.001031, intrinsic_rewards: 0.000023, epsilon: 0.115394, episode:   88\n",
      "frames: 113000, reward: -17.400000, total_loss: 1.745878, forward_loss: 0.000000, inverse_pred_loss: 1.743329, Q_loss: 0.002549, intrinsic_rewards: 0.000035, epsilon: 0.113307, episode:   88\n",
      "frames: 114000, reward: -16.800000, total_loss: 1.788455, forward_loss: 0.000001, inverse_pred_loss: 1.786184, Q_loss: 0.002270, intrinsic_rewards: 0.000075, epsilon: 0.111261, episode:   89\n",
      "frames: 115000, reward: -16.800000, total_loss: 1.776125, forward_loss: 0.000000, inverse_pred_loss: 1.774440, Q_loss: 0.001685, intrinsic_rewards: 0.000013, epsilon: 0.109256, episode:   89\n",
      "frames: 116000, reward: -16.300000, total_loss: 1.777147, forward_loss: 0.000000, inverse_pred_loss: 1.774927, Q_loss: 0.002220, intrinsic_rewards: 0.000009, epsilon: 0.107291, episode:   90\n",
      "frames: 117000, reward: -16.300000, total_loss: 1.759610, forward_loss: 0.000000, inverse_pred_loss: 1.757885, Q_loss: 0.001725, intrinsic_rewards: 0.000019, epsilon: 0.105364, episode:   90\n",
      "frames: 118000, reward: -16.300000, total_loss: 1.812117, forward_loss: 0.000000, inverse_pred_loss: 1.809722, Q_loss: 0.002395, intrinsic_rewards: 0.000029, epsilon: 0.103476, episode:   90\n",
      "frames: 119000, reward: -15.700000, total_loss: 1.747079, forward_loss: 0.000000, inverse_pred_loss: 1.746002, Q_loss: 0.001076, intrinsic_rewards: 0.000005, epsilon: 0.101625, episode:   91\n",
      "frames: 120000, reward: -15.700000, total_loss: 1.774357, forward_loss: 0.000000, inverse_pred_loss: 1.772570, Q_loss: 0.001787, intrinsic_rewards: 0.000009, epsilon: 0.099811, episode:   91\n",
      "frames: 121000, reward: -15.900000, total_loss: 1.766197, forward_loss: 0.000000, inverse_pred_loss: 1.765165, Q_loss: 0.001032, intrinsic_rewards: 0.000010, epsilon: 0.098032, episode:   92\n",
      "frames: 122000, reward: -15.900000, total_loss: 1.768027, forward_loss: 0.000000, inverse_pred_loss: 1.766469, Q_loss: 0.001557, intrinsic_rewards: 0.000016, epsilon: 0.096289, episode:   92\n",
      "frames: 123000, reward: -15.900000, total_loss: 1.789891, forward_loss: 0.000000, inverse_pred_loss: 1.789110, Q_loss: 0.000780, intrinsic_rewards: 0.000011, epsilon: 0.094581, episode:   92\n",
      "frames: 124000, reward: -14.900000, total_loss: 1.836573, forward_loss: 0.000000, inverse_pred_loss: 1.835459, Q_loss: 0.001113, intrinsic_rewards: 0.000008, epsilon: 0.092906, episode:   93\n",
      "frames: 125000, reward: -14.900000, total_loss: 1.760144, forward_loss: 0.000000, inverse_pred_loss: 1.759344, Q_loss: 0.000800, intrinsic_rewards: 0.000012, epsilon: 0.091264, episode:   93\n",
      "frames: 126000, reward: -15.000000, total_loss: 1.766816, forward_loss: 0.000000, inverse_pred_loss: 1.765701, Q_loss: 0.001115, intrinsic_rewards: 0.000014, epsilon: 0.089655, episode:   94\n",
      "frames: 127000, reward: -15.000000, total_loss: 1.779082, forward_loss: 0.000000, inverse_pred_loss: 1.777881, Q_loss: 0.001200, intrinsic_rewards: 0.000009, epsilon: 0.088078, episode:   94\n",
      "frames: 128000, reward: -15.000000, total_loss: 1.780115, forward_loss: 0.000000, inverse_pred_loss: 1.777857, Q_loss: 0.002258, intrinsic_rewards: 0.000020, epsilon: 0.086532, episode:   94\n",
      "frames: 129000, reward: -14.400000, total_loss: 1.791775, forward_loss: 0.000000, inverse_pred_loss: 1.789978, Q_loss: 0.001796, intrinsic_rewards: 0.000013, epsilon: 0.085016, episode:   95\n",
      "frames: 130000, reward: -14.400000, total_loss: 1.777018, forward_loss: 0.000000, inverse_pred_loss: 1.775891, Q_loss: 0.001126, intrinsic_rewards: 0.000004, epsilon: 0.083531, episode:   95\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 131000, reward: -14.400000, total_loss: 1.790152, forward_loss: 0.000000, inverse_pred_loss: 1.783965, Q_loss: 0.006186, intrinsic_rewards: 0.000015, epsilon: 0.082075, episode:   95\n",
      "frames: 132000, reward: -14.200000, total_loss: 1.785104, forward_loss: 0.000000, inverse_pred_loss: 1.783713, Q_loss: 0.001391, intrinsic_rewards: 0.000004, epsilon: 0.080648, episode:   96\n",
      "frames: 133000, reward: -14.200000, total_loss: 1.811727, forward_loss: 0.000000, inverse_pred_loss: 1.810951, Q_loss: 0.000776, intrinsic_rewards: 0.000011, epsilon: 0.079249, episode:   96\n",
      "frames: 134000, reward: -14.100000, total_loss: 1.783053, forward_loss: 0.000000, inverse_pred_loss: 1.781816, Q_loss: 0.001236, intrinsic_rewards: 0.000004, epsilon: 0.077878, episode:   97\n",
      "frames: 135000, reward: -14.100000, total_loss: 1.771201, forward_loss: 0.000000, inverse_pred_loss: 1.770101, Q_loss: 0.001099, intrinsic_rewards: 0.000004, epsilon: 0.076533, episode:   97\n",
      "frames: 136000, reward: -14.100000, total_loss: 1.779893, forward_loss: 0.000000, inverse_pred_loss: 1.779147, Q_loss: 0.000745, intrinsic_rewards: 0.000020, epsilon: 0.075216, episode:   97\n",
      "frames: 137000, reward: -13.500000, total_loss: 1.784338, forward_loss: 0.000000, inverse_pred_loss: 1.783607, Q_loss: 0.000731, intrinsic_rewards: 0.000015, epsilon: 0.073925, episode:   98\n",
      "frames: 138000, reward: -13.500000, total_loss: 1.777946, forward_loss: 0.000000, inverse_pred_loss: 1.776417, Q_loss: 0.001529, intrinsic_rewards: 0.000009, epsilon: 0.072659, episode:   98\n",
      "frames: 139000, reward: -13.500000, total_loss: 1.791986, forward_loss: 0.000000, inverse_pred_loss: 1.791098, Q_loss: 0.000888, intrinsic_rewards: 0.000015, epsilon: 0.071418, episode:   98\n",
      "frames: 140000, reward: -13.500000, total_loss: 1.765260, forward_loss: 0.000000, inverse_pred_loss: 1.762916, Q_loss: 0.002344, intrinsic_rewards: 0.000005, epsilon: 0.070202, episode:   98\n",
      "frames: 141000, reward: -13.200000, total_loss: 1.784551, forward_loss: 0.000000, inverse_pred_loss: 1.779645, Q_loss: 0.004906, intrinsic_rewards: 0.000008, epsilon: 0.069010, episode:   99\n",
      "frames: 142000, reward: -13.200000, total_loss: 1.827140, forward_loss: 0.000000, inverse_pred_loss: 1.826442, Q_loss: 0.000698, intrinsic_rewards: 0.000013, epsilon: 0.067841, episode:   99\n",
      "frames: 143000, reward: -13.400000, total_loss: 1.799772, forward_loss: 0.000000, inverse_pred_loss: 1.798667, Q_loss: 0.001105, intrinsic_rewards: 0.000005, epsilon: 0.066696, episode:  100\n",
      "frames: 144000, reward: -13.400000, total_loss: 1.781751, forward_loss: 0.000000, inverse_pred_loss: 1.778275, Q_loss: 0.003477, intrinsic_rewards: 0.000014, epsilon: 0.065573, episode:  100\n",
      "frames: 145000, reward: -14.200000, total_loss: 1.799323, forward_loss: 0.000000, inverse_pred_loss: 1.793837, Q_loss: 0.005486, intrinsic_rewards: 0.000007, epsilon: 0.064473, episode:  101\n",
      "frames: 146000, reward: -14.200000, total_loss: 1.812806, forward_loss: 0.000000, inverse_pred_loss: 1.811002, Q_loss: 0.001804, intrinsic_rewards: 0.000013, epsilon: 0.063394, episode:  101\n",
      "frames: 147000, reward: -14.200000, total_loss: 1.808463, forward_loss: 0.000000, inverse_pred_loss: 1.805356, Q_loss: 0.003107, intrinsic_rewards: 0.000027, epsilon: 0.062337, episode:  101\n",
      "frames: 148000, reward: -14.000000, total_loss: 1.811698, forward_loss: 0.000000, inverse_pred_loss: 1.810614, Q_loss: 0.001084, intrinsic_rewards: 0.000012, epsilon: 0.061301, episode:  102\n",
      "frames: 149000, reward: -14.000000, total_loss: 1.782943, forward_loss: 0.000000, inverse_pred_loss: 1.781688, Q_loss: 0.001255, intrinsic_rewards: 0.000006, epsilon: 0.060285, episode:  102\n",
      "frames: 150000, reward: -15.000000, total_loss: 1.784218, forward_loss: 0.000000, inverse_pred_loss: 1.783108, Q_loss: 0.001111, intrinsic_rewards: 0.000002, epsilon: 0.059289, episode:  103\n",
      "frames: 151000, reward: -15.000000, total_loss: 1.796225, forward_loss: 0.000000, inverse_pred_loss: 1.794251, Q_loss: 0.001974, intrinsic_rewards: 0.000003, epsilon: 0.058313, episode:  103\n",
      "frames: 152000, reward: -15.000000, total_loss: 1.784757, forward_loss: 0.000000, inverse_pred_loss: 1.783882, Q_loss: 0.000875, intrinsic_rewards: 0.000001, epsilon: 0.057357, episode:  103\n",
      "frames: 153000, reward: -14.900000, total_loss: 1.810875, forward_loss: 0.000000, inverse_pred_loss: 1.800952, Q_loss: 0.009923, intrinsic_rewards: 0.000002, epsilon: 0.056419, episode:  104\n",
      "frames: 154000, reward: -14.900000, total_loss: 1.815796, forward_loss: 0.000000, inverse_pred_loss: 1.814868, Q_loss: 0.000927, intrinsic_rewards: 0.000003, epsilon: 0.055500, episode:  104\n",
      "frames: 155000, reward: -14.900000, total_loss: 1.820898, forward_loss: 0.000000, inverse_pred_loss: 1.819084, Q_loss: 0.001814, intrinsic_rewards: 0.000002, epsilon: 0.054599, episode:  105\n",
      "frames: 156000, reward: -14.900000, total_loss: 1.788372, forward_loss: 0.000000, inverse_pred_loss: 1.787166, Q_loss: 0.001206, intrinsic_rewards: 0.000001, epsilon: 0.053716, episode:  105\n",
      "frames: 157000, reward: -14.900000, total_loss: 1.757455, forward_loss: 0.000000, inverse_pred_loss: 1.756075, Q_loss: 0.001379, intrinsic_rewards: 0.000001, epsilon: 0.052850, episode:  105\n",
      "frames: 158000, reward: -14.900000, total_loss: 1.787027, forward_loss: 0.000000, inverse_pred_loss: 1.786560, Q_loss: 0.000467, intrinsic_rewards: 0.000007, epsilon: 0.052001, episode:  106\n",
      "frames: 159000, reward: -14.900000, total_loss: 1.795778, forward_loss: 0.000000, inverse_pred_loss: 1.795198, Q_loss: 0.000580, intrinsic_rewards: 0.000001, epsilon: 0.051170, episode:  106\n",
      "frames: 160000, reward: -14.900000, total_loss: 1.800435, forward_loss: 0.000000, inverse_pred_loss: 1.799423, Q_loss: 0.001012, intrinsic_rewards: 0.000005, epsilon: 0.050355, episode:  106\n",
      "frames: 161000, reward: -14.800000, total_loss: 1.794195, forward_loss: 0.000000, inverse_pred_loss: 1.793502, Q_loss: 0.000693, intrinsic_rewards: 0.000002, epsilon: 0.049556, episode:  107\n",
      "frames: 162000, reward: -14.800000, total_loss: 1.805881, forward_loss: 0.000000, inverse_pred_loss: 1.805413, Q_loss: 0.000468, intrinsic_rewards: 0.000003, epsilon: 0.048772, episode:  107\n",
      "frames: 163000, reward: -14.800000, total_loss: 1.786917, forward_loss: 0.000000, inverse_pred_loss: 1.783592, Q_loss: 0.003325, intrinsic_rewards: 0.000001, epsilon: 0.048005, episode:  107\n",
      "frames: 164000, reward: -15.400000, total_loss: 1.793195, forward_loss: 0.000000, inverse_pred_loss: 1.790540, Q_loss: 0.002656, intrinsic_rewards: 0.000002, epsilon: 0.047252, episode:  108\n",
      "frames: 165000, reward: -15.400000, total_loss: 1.793054, forward_loss: 0.000000, inverse_pred_loss: 1.792497, Q_loss: 0.000557, intrinsic_rewards: 0.000001, epsilon: 0.046514, episode:  108\n",
      "frames: 166000, reward: -15.800000, total_loss: 1.766352, forward_loss: 0.000000, inverse_pred_loss: 1.765537, Q_loss: 0.000815, intrinsic_rewards: 0.000006, epsilon: 0.045791, episode:  109\n",
      "frames: 167000, reward: -15.800000, total_loss: 1.791687, forward_loss: 0.000000, inverse_pred_loss: 1.789606, Q_loss: 0.002082, intrinsic_rewards: 0.000005, epsilon: 0.045083, episode:  109\n",
      "frames: 168000, reward: -15.800000, total_loss: 1.777699, forward_loss: 0.000000, inverse_pred_loss: 1.776613, Q_loss: 0.001086, intrinsic_rewards: 0.000003, epsilon: 0.044388, episode:  109\n",
      "frames: 169000, reward: -15.300000, total_loss: 1.787935, forward_loss: 0.000000, inverse_pred_loss: 1.787070, Q_loss: 0.000864, intrinsic_rewards: 0.000003, epsilon: 0.043707, episode:  110\n",
      "frames: 170000, reward: -15.300000, total_loss: 1.793533, forward_loss: 0.000000, inverse_pred_loss: 1.792118, Q_loss: 0.001415, intrinsic_rewards: 0.000002, epsilon: 0.043040, episode:  110\n",
      "frames: 171000, reward: -15.300000, total_loss: 1.787395, forward_loss: 0.000000, inverse_pred_loss: 1.786367, Q_loss: 0.001028, intrinsic_rewards: 0.000002, epsilon: 0.042385, episode:  110\n",
      "frames: 172000, reward: -15.300000, total_loss: 1.792542, forward_loss: 0.000000, inverse_pred_loss: 1.791466, Q_loss: 0.001077, intrinsic_rewards: 0.000002, epsilon: 0.041744, episode:  110\n",
      "frames: 173000, reward: -14.500000, total_loss: 1.793450, forward_loss: 0.000000, inverse_pred_loss: 1.792546, Q_loss: 0.000904, intrinsic_rewards: 0.000002, epsilon: 0.041115, episode:  111\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 174000, reward: -14.500000, total_loss: 1.791556, forward_loss: 0.000000, inverse_pred_loss: 1.790742, Q_loss: 0.000814, intrinsic_rewards: 0.000003, epsilon: 0.040499, episode:  111\n",
      "frames: 175000, reward: -14.500000, total_loss: 1.798676, forward_loss: 0.000000, inverse_pred_loss: 1.798123, Q_loss: 0.000553, intrinsic_rewards: 0.000002, epsilon: 0.039895, episode:  111\n",
      "frames: 176000, reward: -14.200000, total_loss: 1.764437, forward_loss: 0.000000, inverse_pred_loss: 1.763243, Q_loss: 0.001193, intrinsic_rewards: 0.000004, epsilon: 0.039303, episode:  112\n",
      "frames: 177000, reward: -14.200000, total_loss: 1.800273, forward_loss: 0.000000, inverse_pred_loss: 1.799220, Q_loss: 0.001053, intrinsic_rewards: 0.000001, epsilon: 0.038723, episode:  112\n",
      "frames: 178000, reward: -14.200000, total_loss: 1.790330, forward_loss: 0.000000, inverse_pred_loss: 1.789276, Q_loss: 0.001054, intrinsic_rewards: 0.000001, epsilon: 0.038154, episode:  112\n",
      "frames: 179000, reward: -13.200000, total_loss: 1.786884, forward_loss: 0.000000, inverse_pred_loss: 1.785389, Q_loss: 0.001494, intrinsic_rewards: 0.000002, epsilon: 0.037597, episode:  113\n",
      "frames: 180000, reward: -13.200000, total_loss: 1.795282, forward_loss: 0.000000, inverse_pred_loss: 1.794507, Q_loss: 0.000775, intrinsic_rewards: 0.000004, epsilon: 0.037050, episode:  113\n",
      "frames: 181000, reward: -13.200000, total_loss: 1.799852, forward_loss: 0.000000, inverse_pred_loss: 1.798437, Q_loss: 0.001416, intrinsic_rewards: 0.000003, epsilon: 0.036515, episode:  113\n",
      "frames: 182000, reward: -12.600000, total_loss: 1.763149, forward_loss: 0.000000, inverse_pred_loss: 1.760307, Q_loss: 0.002842, intrinsic_rewards: 0.000002, epsilon: 0.035990, episode:  114\n",
      "frames: 183000, reward: -12.600000, total_loss: 1.834196, forward_loss: 0.000000, inverse_pred_loss: 1.833583, Q_loss: 0.000613, intrinsic_rewards: 0.000001, epsilon: 0.035475, episode:  114\n",
      "frames: 184000, reward: -12.600000, total_loss: 1.753092, forward_loss: 0.000000, inverse_pred_loss: 1.751733, Q_loss: 0.001359, intrinsic_rewards: 0.000001, epsilon: 0.034971, episode:  114\n",
      "frames: 185000, reward: -12.300000, total_loss: 1.810182, forward_loss: 0.000000, inverse_pred_loss: 1.809376, Q_loss: 0.000806, intrinsic_rewards: 0.000002, epsilon: 0.034476, episode:  115\n",
      "frames: 186000, reward: -12.300000, total_loss: 1.764435, forward_loss: 0.000000, inverse_pred_loss: 1.762812, Q_loss: 0.001624, intrinsic_rewards: 0.000001, epsilon: 0.033992, episode:  115\n",
      "frames: 187000, reward: -12.300000, total_loss: 1.804713, forward_loss: 0.000000, inverse_pred_loss: 1.803612, Q_loss: 0.001101, intrinsic_rewards: 0.000000, epsilon: 0.033517, episode:  115\n",
      "frames: 188000, reward: -12.300000, total_loss: 1.806592, forward_loss: 0.000000, inverse_pred_loss: 1.805822, Q_loss: 0.000770, intrinsic_rewards: 0.000001, epsilon: 0.033051, episode:  116\n",
      "frames: 189000, reward: -12.300000, total_loss: 1.790179, forward_loss: 0.000000, inverse_pred_loss: 1.789192, Q_loss: 0.000987, intrinsic_rewards: 0.000001, epsilon: 0.032594, episode:  116\n",
      "frames: 190000, reward: -12.300000, total_loss: 1.805120, forward_loss: 0.000000, inverse_pred_loss: 1.803252, Q_loss: 0.001868, intrinsic_rewards: 0.000001, epsilon: 0.032147, episode:  116\n",
      "frames: 191000, reward: -12.100000, total_loss: 1.790510, forward_loss: 0.000000, inverse_pred_loss: 1.789838, Q_loss: 0.000672, intrinsic_rewards: 0.000004, epsilon: 0.031709, episode:  117\n",
      "frames: 192000, reward: -12.100000, total_loss: 1.770184, forward_loss: 0.000000, inverse_pred_loss: 1.768891, Q_loss: 0.001293, intrinsic_rewards: 0.000001, epsilon: 0.031279, episode:  117\n",
      "frames: 193000, reward: -12.100000, total_loss: 1.739756, forward_loss: 0.000000, inverse_pred_loss: 1.739053, Q_loss: 0.000702, intrinsic_rewards: 0.000001, epsilon: 0.030857, episode:  117\n",
      "frames: 194000, reward: -11.600000, total_loss: 1.782927, forward_loss: 0.000000, inverse_pred_loss: 1.779941, Q_loss: 0.002986, intrinsic_rewards: 0.000001, epsilon: 0.030444, episode:  118\n",
      "frames: 195000, reward: -11.600000, total_loss: 1.788143, forward_loss: 0.000000, inverse_pred_loss: 1.787157, Q_loss: 0.000986, intrinsic_rewards: 0.000000, epsilon: 0.030039, episode:  118\n",
      "frames: 196000, reward: -11.600000, total_loss: 1.799849, forward_loss: 0.000000, inverse_pred_loss: 1.799133, Q_loss: 0.000716, intrinsic_rewards: 0.000001, epsilon: 0.029643, episode:  118\n",
      "frames: 197000, reward: -11.600000, total_loss: 1.785025, forward_loss: 0.000000, inverse_pred_loss: 1.784210, Q_loss: 0.000815, intrinsic_rewards: 0.000000, epsilon: 0.029254, episode:  119\n",
      "frames: 198000, reward: -11.600000, total_loss: 1.797714, forward_loss: 0.000000, inverse_pred_loss: 1.796519, Q_loss: 0.001195, intrinsic_rewards: 0.000001, epsilon: 0.028872, episode:  119\n",
      "frames: 199000, reward: -11.600000, total_loss: 1.795871, forward_loss: 0.000000, inverse_pred_loss: 1.795175, Q_loss: 0.000696, intrinsic_rewards: 0.000002, epsilon: 0.028499, episode:  119\n",
      "frames: 200000, reward: -11.300000, total_loss: 1.794330, forward_loss: 0.000000, inverse_pred_loss: 1.793339, Q_loss: 0.000991, intrinsic_rewards: 0.000007, epsilon: 0.028132, episode:  120\n",
      "frames: 201000, reward: -11.300000, total_loss: 1.779455, forward_loss: 0.000000, inverse_pred_loss: 1.778172, Q_loss: 0.001283, intrinsic_rewards: 0.000002, epsilon: 0.027773, episode:  120\n",
      "frames: 202000, reward: -11.300000, total_loss: 1.822630, forward_loss: 0.000000, inverse_pred_loss: 1.821741, Q_loss: 0.000889, intrinsic_rewards: 0.000000, epsilon: 0.027421, episode:  120\n",
      "frames: 203000, reward: -11.300000, total_loss: 1.761352, forward_loss: 0.000000, inverse_pred_loss: 1.759684, Q_loss: 0.001668, intrinsic_rewards: 0.000001, epsilon: 0.027077, episode:  120\n",
      "frames: 204000, reward: -10.200000, total_loss: 1.777636, forward_loss: 0.000000, inverse_pred_loss: 1.776566, Q_loss: 0.001070, intrinsic_rewards: 0.000000, epsilon: 0.026738, episode:  121\n",
      "frames: 205000, reward: -10.200000, total_loss: 1.794167, forward_loss: 0.000000, inverse_pred_loss: 1.792215, Q_loss: 0.001952, intrinsic_rewards: 0.000001, epsilon: 0.026407, episode:  121\n",
      "frames: 206000, reward: -10.200000, total_loss: 1.793106, forward_loss: 0.000000, inverse_pred_loss: 1.789612, Q_loss: 0.003494, intrinsic_rewards: 0.000001, epsilon: 0.026082, episode:  121\n",
      "frames: 207000, reward: -9.200000, total_loss: 1.768090, forward_loss: 0.000000, inverse_pred_loss: 1.767539, Q_loss: 0.000550, intrinsic_rewards: 0.000001, epsilon: 0.025764, episode:  122\n",
      "frames: 208000, reward: -9.200000, total_loss: 1.796478, forward_loss: 0.000000, inverse_pred_loss: 1.793513, Q_loss: 0.002965, intrinsic_rewards: 0.000001, epsilon: 0.025451, episode:  122\n",
      "frames: 209000, reward: -9.200000, total_loss: 1.781540, forward_loss: 0.000000, inverse_pred_loss: 1.780763, Q_loss: 0.000777, intrinsic_rewards: 0.000001, epsilon: 0.025146, episode:  122\n",
      "frames: 210000, reward: -9.400000, total_loss: 1.776472, forward_loss: 0.000000, inverse_pred_loss: 1.774830, Q_loss: 0.001642, intrinsic_rewards: 0.000001, epsilon: 0.024846, episode:  123\n",
      "frames: 211000, reward: -9.400000, total_loss: 1.758686, forward_loss: 0.000000, inverse_pred_loss: 1.757780, Q_loss: 0.000906, intrinsic_rewards: 0.000001, epsilon: 0.024552, episode:  123\n",
      "frames: 212000, reward: -9.400000, total_loss: 1.781012, forward_loss: 0.000000, inverse_pred_loss: 1.779575, Q_loss: 0.001437, intrinsic_rewards: 0.000001, epsilon: 0.024264, episode:  123\n",
      "frames: 213000, reward: -9.300000, total_loss: 1.802211, forward_loss: 0.000000, inverse_pred_loss: 1.801729, Q_loss: 0.000482, intrinsic_rewards: 0.000000, epsilon: 0.023981, episode:  124\n",
      "frames: 214000, reward: -9.300000, total_loss: 1.768956, forward_loss: 0.000000, inverse_pred_loss: 1.768100, Q_loss: 0.000857, intrinsic_rewards: 0.000001, epsilon: 0.023704, episode:  124\n",
      "frames: 215000, reward: -9.300000, total_loss: 1.780342, forward_loss: 0.000000, inverse_pred_loss: 1.777317, Q_loss: 0.003025, intrinsic_rewards: 0.000000, epsilon: 0.023433, episode:  124\n",
      "frames: 216000, reward: -9.300000, total_loss: 1.788445, forward_loss: 0.000000, inverse_pred_loss: 1.787707, Q_loss: 0.000737, intrinsic_rewards: 0.000001, epsilon: 0.023167, episode:  124\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 217000, reward: -8.600000, total_loss: 1.788790, forward_loss: 0.000000, inverse_pred_loss: 1.787504, Q_loss: 0.001286, intrinsic_rewards: 0.000001, epsilon: 0.022906, episode:  125\n",
      "frames: 218000, reward: -8.600000, total_loss: 1.794479, forward_loss: 0.000000, inverse_pred_loss: 1.793330, Q_loss: 0.001148, intrinsic_rewards: 0.000000, epsilon: 0.022651, episode:  125\n",
      "frames: 219000, reward: -8.600000, total_loss: 1.765303, forward_loss: 0.000000, inverse_pred_loss: 1.764317, Q_loss: 0.000985, intrinsic_rewards: 0.000001, epsilon: 0.022400, episode:  125\n",
      "frames: 220000, reward: -7.000000, total_loss: 1.766065, forward_loss: 0.000000, inverse_pred_loss: 1.765090, Q_loss: 0.000974, intrinsic_rewards: 0.000002, epsilon: 0.022155, episode:  126\n",
      "frames: 221000, reward: -7.000000, total_loss: 1.813326, forward_loss: 0.000000, inverse_pred_loss: 1.812648, Q_loss: 0.000678, intrinsic_rewards: 0.000001, epsilon: 0.021914, episode:  126\n",
      "frames: 222000, reward: -7.200000, total_loss: 1.799960, forward_loss: 0.000000, inverse_pred_loss: 1.799158, Q_loss: 0.000802, intrinsic_rewards: 0.000001, epsilon: 0.021678, episode:  127\n",
      "frames: 223000, reward: -7.200000, total_loss: 1.806090, forward_loss: 0.000000, inverse_pred_loss: 1.805073, Q_loss: 0.001017, intrinsic_rewards: 0.000001, epsilon: 0.021447, episode:  127\n",
      "frames: 224000, reward: -7.200000, total_loss: 1.788549, forward_loss: 0.000000, inverse_pred_loss: 1.787955, Q_loss: 0.000594, intrinsic_rewards: 0.000000, epsilon: 0.021220, episode:  127\n",
      "frames: 225000, reward: -7.200000, total_loss: 1.766840, forward_loss: 0.000000, inverse_pred_loss: 1.765969, Q_loss: 0.000871, intrinsic_rewards: 0.000001, epsilon: 0.020998, episode:  128\n",
      "frames: 226000, reward: -7.200000, total_loss: 1.811132, forward_loss: 0.000000, inverse_pred_loss: 1.810326, Q_loss: 0.000807, intrinsic_rewards: 0.000001, epsilon: 0.020780, episode:  128\n",
      "frames: 227000, reward: -7.200000, total_loss: 1.769119, forward_loss: 0.000000, inverse_pred_loss: 1.765667, Q_loss: 0.003451, intrinsic_rewards: 0.000001, epsilon: 0.020567, episode:  128\n",
      "frames: 228000, reward: -7.200000, total_loss: 1.761170, forward_loss: 0.000000, inverse_pred_loss: 1.760120, Q_loss: 0.001050, intrinsic_rewards: 0.000001, epsilon: 0.020357, episode:  128\n",
      "frames: 229000, reward: -6.400000, total_loss: 1.770747, forward_loss: 0.000000, inverse_pred_loss: 1.769867, Q_loss: 0.000880, intrinsic_rewards: 0.000001, epsilon: 0.020152, episode:  129\n",
      "frames: 230000, reward: -6.400000, total_loss: 1.798069, forward_loss: 0.000000, inverse_pred_loss: 1.797198, Q_loss: 0.000871, intrinsic_rewards: 0.000002, epsilon: 0.019951, episode:  129\n",
      "frames: 231000, reward: -6.400000, total_loss: 1.779522, forward_loss: 0.000000, inverse_pred_loss: 1.778479, Q_loss: 0.001044, intrinsic_rewards: 0.000001, epsilon: 0.019754, episode:  129\n",
      "frames: 232000, reward: -6.400000, total_loss: 1.785955, forward_loss: 0.000000, inverse_pred_loss: 1.785004, Q_loss: 0.000951, intrinsic_rewards: 0.000001, epsilon: 0.019561, episode:  129\n",
      "frames: 233000, reward: -5.700000, total_loss: 1.794293, forward_loss: 0.000000, inverse_pred_loss: 1.792845, Q_loss: 0.001447, intrinsic_rewards: 0.000001, epsilon: 0.019372, episode:  130\n",
      "frames: 234000, reward: -5.700000, total_loss: 1.765609, forward_loss: 0.000000, inverse_pred_loss: 1.764676, Q_loss: 0.000932, intrinsic_rewards: 0.000001, epsilon: 0.019186, episode:  130\n",
      "frames: 235000, reward: -5.700000, total_loss: 1.811652, forward_loss: 0.000000, inverse_pred_loss: 1.810774, Q_loss: 0.000878, intrinsic_rewards: 0.000001, epsilon: 0.019004, episode:  130\n",
      "frames: 236000, reward: -6.000000, total_loss: 1.774823, forward_loss: 0.000000, inverse_pred_loss: 1.773952, Q_loss: 0.000871, intrinsic_rewards: 0.000003, epsilon: 0.018826, episode:  131\n",
      "frames: 237000, reward: -6.000000, total_loss: 1.789540, forward_loss: 0.000000, inverse_pred_loss: 1.787897, Q_loss: 0.001643, intrinsic_rewards: 0.000001, epsilon: 0.018651, episode:  131\n",
      "frames: 238000, reward: -7.200000, total_loss: 1.750601, forward_loss: 0.000000, inverse_pred_loss: 1.749884, Q_loss: 0.000718, intrinsic_rewards: 0.000001, epsilon: 0.018480, episode:  132\n",
      "frames: 239000, reward: -7.200000, total_loss: 1.785422, forward_loss: 0.000000, inverse_pred_loss: 1.782864, Q_loss: 0.002558, intrinsic_rewards: 0.000001, epsilon: 0.018312, episode:  132\n",
      "frames: 240000, reward: -7.200000, total_loss: 1.740214, forward_loss: 0.000000, inverse_pred_loss: 1.739653, Q_loss: 0.000560, intrinsic_rewards: 0.000003, epsilon: 0.018147, episode:  132\n",
      "frames: 241000, reward: -7.200000, total_loss: 1.764836, forward_loss: 0.000000, inverse_pred_loss: 1.763453, Q_loss: 0.001383, intrinsic_rewards: 0.000001, epsilon: 0.017986, episode:  132\n",
      "frames: 242000, reward: -6.600000, total_loss: 1.796082, forward_loss: 0.000000, inverse_pred_loss: 1.794935, Q_loss: 0.001148, intrinsic_rewards: 0.000002, epsilon: 0.017828, episode:  133\n",
      "frames: 243000, reward: -6.600000, total_loss: 1.797701, forward_loss: 0.000000, inverse_pred_loss: 1.796446, Q_loss: 0.001256, intrinsic_rewards: 0.000002, epsilon: 0.017673, episode:  133\n",
      "frames: 244000, reward: -6.600000, total_loss: 1.787629, forward_loss: 0.000000, inverse_pred_loss: 1.787322, Q_loss: 0.000307, intrinsic_rewards: 0.000001, epsilon: 0.017521, episode:  133\n",
      "frames: 245000, reward: -6.600000, total_loss: 1.786616, forward_loss: 0.000000, inverse_pred_loss: 1.786028, Q_loss: 0.000588, intrinsic_rewards: 0.000001, epsilon: 0.017372, episode:  134\n",
      "frames: 246000, reward: -6.600000, total_loss: 1.781018, forward_loss: 0.000000, inverse_pred_loss: 1.780349, Q_loss: 0.000668, intrinsic_rewards: 0.000006, epsilon: 0.017226, episode:  134\n",
      "frames: 247000, reward: -6.600000, total_loss: 1.800613, forward_loss: 0.000000, inverse_pred_loss: 1.799186, Q_loss: 0.001427, intrinsic_rewards: 0.000005, epsilon: 0.017083, episode:  134\n",
      "frames: 248000, reward: -6.600000, total_loss: 1.753349, forward_loss: 0.000000, inverse_pred_loss: 1.752822, Q_loss: 0.000527, intrinsic_rewards: 0.000003, epsilon: 0.016943, episode:  134\n",
      "frames: 249000, reward: -7.100000, total_loss: 1.771129, forward_loss: 0.000000, inverse_pred_loss: 1.770286, Q_loss: 0.000843, intrinsic_rewards: 0.000002, epsilon: 0.016805, episode:  135\n",
      "frames: 250000, reward: -7.100000, total_loss: 1.822477, forward_loss: 0.000000, inverse_pred_loss: 1.821528, Q_loss: 0.000949, intrinsic_rewards: 0.000003, epsilon: 0.016671, episode:  135\n",
      "frames: 251000, reward: -7.100000, total_loss: 1.775695, forward_loss: 0.000000, inverse_pred_loss: 1.775237, Q_loss: 0.000458, intrinsic_rewards: 0.000002, epsilon: 0.016538, episode:  135\n",
      "frames: 252000, reward: -8.200000, total_loss: 1.801768, forward_loss: 0.000000, inverse_pred_loss: 1.800896, Q_loss: 0.000872, intrinsic_rewards: 0.000003, epsilon: 0.016409, episode:  136\n",
      "frames: 253000, reward: -8.200000, total_loss: 1.773870, forward_loss: 0.000000, inverse_pred_loss: 1.772951, Q_loss: 0.000918, intrinsic_rewards: 0.000001, epsilon: 0.016282, episode:  136\n",
      "frames: 254000, reward: -8.200000, total_loss: 1.741431, forward_loss: 0.000000, inverse_pred_loss: 1.740267, Q_loss: 0.001164, intrinsic_rewards: 0.000001, epsilon: 0.016158, episode:  136\n",
      "frames: 255000, reward: -8.200000, total_loss: 1.776382, forward_loss: 0.000000, inverse_pred_loss: 1.775560, Q_loss: 0.000823, intrinsic_rewards: 0.000001, epsilon: 0.016036, episode:  136\n",
      "frames: 256000, reward: -6.400000, total_loss: 1.776874, forward_loss: 0.000000, inverse_pred_loss: 1.775228, Q_loss: 0.001646, intrinsic_rewards: 0.000002, epsilon: 0.015916, episode:  137\n",
      "frames: 257000, reward: -6.400000, total_loss: 1.765632, forward_loss: 0.000000, inverse_pred_loss: 1.764921, Q_loss: 0.000711, intrinsic_rewards: 0.000002, epsilon: 0.015799, episode:  137\n",
      "frames: 258000, reward: -6.400000, total_loss: 1.794342, forward_loss: 0.000000, inverse_pred_loss: 1.793114, Q_loss: 0.001228, intrinsic_rewards: 0.000001, epsilon: 0.015684, episode:  137\n",
      "frames: 259000, reward: -6.500000, total_loss: 1.784988, forward_loss: 0.000000, inverse_pred_loss: 1.784009, Q_loss: 0.000978, intrinsic_rewards: 0.000003, epsilon: 0.015572, episode:  138\n",
      "frames: 260000, reward: -6.500000, total_loss: 1.765798, forward_loss: 0.000000, inverse_pred_loss: 1.764209, Q_loss: 0.001588, intrinsic_rewards: 0.000003, epsilon: 0.015461, episode:  138\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 261000, reward: -6.500000, total_loss: 1.777654, forward_loss: 0.000000, inverse_pred_loss: 1.775996, Q_loss: 0.001658, intrinsic_rewards: 0.000001, epsilon: 0.015353, episode:  138\n",
      "frames: 262000, reward: -6.900000, total_loss: 1.745703, forward_loss: 0.000000, inverse_pred_loss: 1.744492, Q_loss: 0.001211, intrinsic_rewards: 0.000003, epsilon: 0.015247, episode:  139\n",
      "frames: 263000, reward: -6.900000, total_loss: 1.791533, forward_loss: 0.000000, inverse_pred_loss: 1.790675, Q_loss: 0.000858, intrinsic_rewards: 0.000001, epsilon: 0.015143, episode:  139\n",
      "frames: 264000, reward: -6.900000, total_loss: 1.756275, forward_loss: 0.000000, inverse_pred_loss: 1.754711, Q_loss: 0.001565, intrinsic_rewards: 0.000001, epsilon: 0.015042, episode:  139\n",
      "frames: 265000, reward: -6.900000, total_loss: 1.771787, forward_loss: 0.000000, inverse_pred_loss: 1.771319, Q_loss: 0.000468, intrinsic_rewards: 0.000001, epsilon: 0.014942, episode:  139\n",
      "frames: 266000, reward: -7.800000, total_loss: 1.790649, forward_loss: 0.000000, inverse_pred_loss: 1.788430, Q_loss: 0.002219, intrinsic_rewards: 0.000000, epsilon: 0.014844, episode:  140\n",
      "frames: 267000, reward: -7.800000, total_loss: 1.783040, forward_loss: 0.000000, inverse_pred_loss: 1.782021, Q_loss: 0.001018, intrinsic_rewards: 0.000001, epsilon: 0.014748, episode:  140\n",
      "frames: 268000, reward: -7.800000, total_loss: 1.784418, forward_loss: 0.000000, inverse_pred_loss: 1.782741, Q_loss: 0.001677, intrinsic_rewards: 0.000001, epsilon: 0.014654, episode:  140\n",
      "frames: 269000, reward: -8.400000, total_loss: 1.772341, forward_loss: 0.000000, inverse_pred_loss: 1.766516, Q_loss: 0.005825, intrinsic_rewards: 0.000001, epsilon: 0.014562, episode:  141\n",
      "frames: 270000, reward: -8.400000, total_loss: 1.777287, forward_loss: 0.000000, inverse_pred_loss: 1.775586, Q_loss: 0.001701, intrinsic_rewards: 0.000001, epsilon: 0.014471, episode:  141\n",
      "frames: 271000, reward: -8.400000, total_loss: 1.799792, forward_loss: 0.000000, inverse_pred_loss: 1.798821, Q_loss: 0.000971, intrinsic_rewards: 0.000001, epsilon: 0.014383, episode:  141\n",
      "frames: 272000, reward: -7.700000, total_loss: 1.781707, forward_loss: 0.000000, inverse_pred_loss: 1.780936, Q_loss: 0.000771, intrinsic_rewards: 0.000001, epsilon: 0.014296, episode:  142\n",
      "frames: 273000, reward: -7.700000, total_loss: 1.770910, forward_loss: 0.000000, inverse_pred_loss: 1.769720, Q_loss: 0.001190, intrinsic_rewards: 0.000000, epsilon: 0.014211, episode:  142\n",
      "frames: 274000, reward: -7.700000, total_loss: 1.769020, forward_loss: 0.000000, inverse_pred_loss: 1.768093, Q_loss: 0.000927, intrinsic_rewards: 0.000001, epsilon: 0.014128, episode:  142\n",
      "frames: 275000, reward: -7.700000, total_loss: 1.766796, forward_loss: 0.000000, inverse_pred_loss: 1.766250, Q_loss: 0.000546, intrinsic_rewards: 0.000003, epsilon: 0.014046, episode:  142\n",
      "frames: 276000, reward: -7.600000, total_loss: 1.779591, forward_loss: 0.000000, inverse_pred_loss: 1.778544, Q_loss: 0.001047, intrinsic_rewards: 0.000001, epsilon: 0.013966, episode:  143\n",
      "frames: 277000, reward: -7.600000, total_loss: 1.781714, forward_loss: 0.000000, inverse_pred_loss: 1.780822, Q_loss: 0.000892, intrinsic_rewards: 0.000002, epsilon: 0.013887, episode:  143\n",
      "frames: 278000, reward: -8.000000, total_loss: 1.793308, forward_loss: 0.000000, inverse_pred_loss: 1.791769, Q_loss: 0.001539, intrinsic_rewards: 0.000001, epsilon: 0.013810, episode:  144\n",
      "frames: 279000, reward: -8.000000, total_loss: 1.770384, forward_loss: 0.000000, inverse_pred_loss: 1.769530, Q_loss: 0.000853, intrinsic_rewards: 0.000002, epsilon: 0.013735, episode:  144\n",
      "frames: 280000, reward: -8.000000, total_loss: 1.767060, forward_loss: 0.000000, inverse_pred_loss: 1.766087, Q_loss: 0.000973, intrinsic_rewards: 0.000002, epsilon: 0.013661, episode:  144\n",
      "frames: 281000, reward: -8.000000, total_loss: 1.778231, forward_loss: 0.000000, inverse_pred_loss: 1.777194, Q_loss: 0.001037, intrinsic_rewards: 0.000001, epsilon: 0.013588, episode:  145\n",
      "frames: 282000, reward: -8.000000, total_loss: 1.765651, forward_loss: 0.000000, inverse_pred_loss: 1.764841, Q_loss: 0.000810, intrinsic_rewards: 0.000002, epsilon: 0.013517, episode:  145\n",
      "frames: 283000, reward: -8.000000, total_loss: 1.779101, forward_loss: 0.000000, inverse_pred_loss: 1.778444, Q_loss: 0.000657, intrinsic_rewards: 0.000000, epsilon: 0.013448, episode:  145\n",
      "frames: 284000, reward: -8.000000, total_loss: 1.769423, forward_loss: 0.000000, inverse_pred_loss: 1.768839, Q_loss: 0.000584, intrinsic_rewards: 0.000001, epsilon: 0.013379, episode:  145\n",
      "frames: 285000, reward: -7.600000, total_loss: 1.771718, forward_loss: 0.000000, inverse_pred_loss: 1.770970, Q_loss: 0.000748, intrinsic_rewards: 0.000002, epsilon: 0.013313, episode:  146\n",
      "frames: 286000, reward: -7.600000, total_loss: 1.761441, forward_loss: 0.000000, inverse_pred_loss: 1.760498, Q_loss: 0.000942, intrinsic_rewards: 0.000005, epsilon: 0.013247, episode:  146\n",
      "frames: 287000, reward: -7.600000, total_loss: 1.822428, forward_loss: 0.000000, inverse_pred_loss: 1.822015, Q_loss: 0.000413, intrinsic_rewards: 0.000000, epsilon: 0.013183, episode:  146\n",
      "frames: 288000, reward: -7.600000, total_loss: 1.775602, forward_loss: 0.000000, inverse_pred_loss: 1.775111, Q_loss: 0.000491, intrinsic_rewards: 0.000001, epsilon: 0.013120, episode:  146\n",
      "frames: 289000, reward: -8.000000, total_loss: 1.756425, forward_loss: 0.000000, inverse_pred_loss: 1.754813, Q_loss: 0.001611, intrinsic_rewards: 0.000001, epsilon: 0.013058, episode:  147\n",
      "frames: 290000, reward: -8.000000, total_loss: 1.784747, forward_loss: 0.000000, inverse_pred_loss: 1.783390, Q_loss: 0.001357, intrinsic_rewards: 0.000000, epsilon: 0.012997, episode:  147\n",
      "frames: 291000, reward: -8.000000, total_loss: 1.787663, forward_loss: 0.000000, inverse_pred_loss: 1.786959, Q_loss: 0.000704, intrinsic_rewards: 0.000003, epsilon: 0.012938, episode:  147\n",
      "frames: 292000, reward: -8.000000, total_loss: 1.768821, forward_loss: 0.000000, inverse_pred_loss: 1.767212, Q_loss: 0.001610, intrinsic_rewards: 0.000000, epsilon: 0.012880, episode:  147\n",
      "frames: 293000, reward: -8.100000, total_loss: 1.781483, forward_loss: 0.000000, inverse_pred_loss: 1.779029, Q_loss: 0.002454, intrinsic_rewards: 0.000001, epsilon: 0.012823, episode:  148\n",
      "frames: 294000, reward: -8.100000, total_loss: 1.756944, forward_loss: 0.000000, inverse_pred_loss: 1.756105, Q_loss: 0.000839, intrinsic_rewards: 0.000001, epsilon: 0.012767, episode:  148\n",
      "frames: 295000, reward: -8.100000, total_loss: 1.726927, forward_loss: 0.000000, inverse_pred_loss: 1.726231, Q_loss: 0.000696, intrinsic_rewards: 0.000002, epsilon: 0.012712, episode:  148\n",
      "frames: 296000, reward: -7.800000, total_loss: 1.728332, forward_loss: 0.000000, inverse_pred_loss: 1.727464, Q_loss: 0.000868, intrinsic_rewards: 0.000001, epsilon: 0.012658, episode:  149\n",
      "frames: 297000, reward: -7.800000, total_loss: 1.757719, forward_loss: 0.000000, inverse_pred_loss: 1.756977, Q_loss: 0.000741, intrinsic_rewards: 0.000000, epsilon: 0.012606, episode:  149\n",
      "frames: 298000, reward: -7.800000, total_loss: 1.771898, forward_loss: 0.000000, inverse_pred_loss: 1.771285, Q_loss: 0.000613, intrinsic_rewards: 0.000001, epsilon: 0.012554, episode:  149\n",
      "frames: 299000, reward: -7.200000, total_loss: 1.783178, forward_loss: 0.000000, inverse_pred_loss: 1.782311, Q_loss: 0.000867, intrinsic_rewards: 0.000004, epsilon: 0.012504, episode:  150\n",
      "frames: 300000, reward: -7.200000, total_loss: 1.755859, forward_loss: 0.000000, inverse_pred_loss: 1.755200, Q_loss: 0.000659, intrinsic_rewards: 0.000001, epsilon: 0.012454, episode:  150\n",
      "frames: 301000, reward: -7.200000, total_loss: 1.790520, forward_loss: 0.000000, inverse_pred_loss: 1.789707, Q_loss: 0.000812, intrinsic_rewards: 0.000003, epsilon: 0.012405, episode:  150\n",
      "frames: 302000, reward: -7.200000, total_loss: 1.767934, forward_loss: 0.000000, inverse_pred_loss: 1.766634, Q_loss: 0.001300, intrinsic_rewards: 0.000001, epsilon: 0.012358, episode:  150\n",
      "frames: 303000, reward: -6.200000, total_loss: 1.804228, forward_loss: 0.000000, inverse_pred_loss: 1.803629, Q_loss: 0.000599, intrinsic_rewards: 0.000000, epsilon: 0.012311, episode:  151\n",
      "frames: 304000, reward: -6.200000, total_loss: 1.747774, forward_loss: 0.000000, inverse_pred_loss: 1.746964, Q_loss: 0.000809, intrinsic_rewards: 0.000001, epsilon: 0.012265, episode:  151\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 305000, reward: -6.200000, total_loss: 1.789591, forward_loss: 0.000000, inverse_pred_loss: 1.789016, Q_loss: 0.000575, intrinsic_rewards: 0.000002, epsilon: 0.012220, episode:  151\n",
      "frames: 306000, reward: -6.200000, total_loss: 1.728986, forward_loss: 0.000000, inverse_pred_loss: 1.728540, Q_loss: 0.000446, intrinsic_rewards: 0.000001, epsilon: 0.012176, episode:  151\n",
      "frames: 307000, reward: -5.500000, total_loss: 1.780451, forward_loss: 0.000000, inverse_pred_loss: 1.779760, Q_loss: 0.000691, intrinsic_rewards: 0.000001, epsilon: 0.012133, episode:  152\n",
      "frames: 308000, reward: -5.500000, total_loss: 1.814164, forward_loss: 0.000000, inverse_pred_loss: 1.813649, Q_loss: 0.000516, intrinsic_rewards: 0.000001, epsilon: 0.012091, episode:  152\n",
      "frames: 309000, reward: -5.500000, total_loss: 1.765638, forward_loss: 0.000000, inverse_pred_loss: 1.764798, Q_loss: 0.000841, intrinsic_rewards: 0.000002, epsilon: 0.012050, episode:  152\n",
      "frames: 310000, reward: -5.600000, total_loss: 1.774917, forward_loss: 0.000000, inverse_pred_loss: 1.773768, Q_loss: 0.001149, intrinsic_rewards: 0.000003, epsilon: 0.012009, episode:  153\n",
      "frames: 311000, reward: -5.600000, total_loss: 1.792969, forward_loss: 0.000000, inverse_pred_loss: 1.791901, Q_loss: 0.001068, intrinsic_rewards: 0.000002, epsilon: 0.011969, episode:  153\n",
      "frames: 312000, reward: -5.600000, total_loss: 1.747209, forward_loss: 0.000000, inverse_pred_loss: 1.745484, Q_loss: 0.001725, intrinsic_rewards: 0.000005, epsilon: 0.011930, episode:  153\n",
      "frames: 313000, reward: -5.600000, total_loss: 1.787470, forward_loss: 0.000000, inverse_pred_loss: 1.780083, Q_loss: 0.007387, intrinsic_rewards: 0.000002, epsilon: 0.011892, episode:  154\n",
      "frames: 314000, reward: -5.600000, total_loss: 1.793578, forward_loss: 0.000000, inverse_pred_loss: 1.793087, Q_loss: 0.000491, intrinsic_rewards: 0.000004, epsilon: 0.011855, episode:  154\n",
      "frames: 315000, reward: -5.900000, total_loss: 1.805618, forward_loss: 0.000000, inverse_pred_loss: 1.805175, Q_loss: 0.000443, intrinsic_rewards: 0.000002, epsilon: 0.011818, episode:  155\n",
      "frames: 316000, reward: -5.900000, total_loss: 1.801565, forward_loss: 0.000000, inverse_pred_loss: 1.800494, Q_loss: 0.001071, intrinsic_rewards: 0.000009, epsilon: 0.011782, episode:  155\n",
      "frames: 317000, reward: -5.900000, total_loss: 1.774660, forward_loss: 0.000000, inverse_pred_loss: 1.774177, Q_loss: 0.000483, intrinsic_rewards: 0.000003, epsilon: 0.011747, episode:  155\n",
      "frames: 318000, reward: -6.400000, total_loss: 1.797860, forward_loss: 0.000000, inverse_pred_loss: 1.795412, Q_loss: 0.002448, intrinsic_rewards: 0.000002, epsilon: 0.011712, episode:  156\n",
      "frames: 319000, reward: -6.400000, total_loss: 1.808354, forward_loss: 0.000000, inverse_pred_loss: 1.807259, Q_loss: 0.001095, intrinsic_rewards: 0.000004, epsilon: 0.011678, episode:  156\n",
      "frames: 320000, reward: -7.600000, total_loss: 1.777994, forward_loss: 0.000000, inverse_pred_loss: 1.772050, Q_loss: 0.005944, intrinsic_rewards: 0.000002, epsilon: 0.011645, episode:  157\n",
      "frames: 321000, reward: -7.600000, total_loss: 1.775886, forward_loss: 0.000000, inverse_pred_loss: 1.773850, Q_loss: 0.002036, intrinsic_rewards: 0.000004, epsilon: 0.011612, episode:  157\n",
      "frames: 322000, reward: -7.600000, total_loss: 1.795206, forward_loss: 0.000000, inverse_pred_loss: 1.794659, Q_loss: 0.000548, intrinsic_rewards: 0.000001, epsilon: 0.011580, episode:  157\n",
      "frames: 323000, reward: -7.600000, total_loss: 1.810475, forward_loss: 0.000000, inverse_pred_loss: 1.808150, Q_loss: 0.002325, intrinsic_rewards: 0.000012, epsilon: 0.011549, episode:  157\n",
      "frames: 324000, reward: -6.600000, total_loss: 1.820309, forward_loss: 0.000000, inverse_pred_loss: 1.819783, Q_loss: 0.000526, intrinsic_rewards: 0.000049, epsilon: 0.011518, episode:  158\n",
      "frames: 325000, reward: -6.600000, total_loss: 1.791190, forward_loss: 0.000000, inverse_pred_loss: 1.790827, Q_loss: 0.000363, intrinsic_rewards: 0.000004, epsilon: 0.011488, episode:  158\n",
      "frames: 326000, reward: -6.600000, total_loss: 1.797915, forward_loss: 0.000000, inverse_pred_loss: 1.796995, Q_loss: 0.000920, intrinsic_rewards: 0.000002, epsilon: 0.011459, episode:  158\n",
      "frames: 327000, reward: -6.700000, total_loss: 1.762224, forward_loss: 0.000000, inverse_pred_loss: 1.761837, Q_loss: 0.000387, intrinsic_rewards: 0.000009, epsilon: 0.011430, episode:  159\n",
      "frames: 328000, reward: -6.700000, total_loss: 1.811441, forward_loss: 0.000000, inverse_pred_loss: 1.810673, Q_loss: 0.000768, intrinsic_rewards: 0.000003, epsilon: 0.011402, episode:  159\n",
      "frames: 329000, reward: -6.700000, total_loss: 1.826219, forward_loss: 0.000000, inverse_pred_loss: 1.825355, Q_loss: 0.000864, intrinsic_rewards: 0.000000, epsilon: 0.011374, episode:  159\n",
      "frames: 330000, reward: -6.700000, total_loss: 1.791313, forward_loss: 0.000000, inverse_pred_loss: 1.790477, Q_loss: 0.000835, intrinsic_rewards: 0.000001, epsilon: 0.011347, episode:  159\n",
      "frames: 331000, reward: -6.400000, total_loss: 1.796236, forward_loss: 0.000000, inverse_pred_loss: 1.795763, Q_loss: 0.000472, intrinsic_rewards: 0.000001, epsilon: 0.011320, episode:  160\n",
      "frames: 332000, reward: -6.400000, total_loss: 1.804315, forward_loss: 0.000000, inverse_pred_loss: 1.803204, Q_loss: 0.001111, intrinsic_rewards: 0.000002, epsilon: 0.011294, episode:  160\n",
      "frames: 333000, reward: -6.400000, total_loss: 1.738320, forward_loss: 0.000000, inverse_pred_loss: 1.737915, Q_loss: 0.000405, intrinsic_rewards: 0.000002, epsilon: 0.011268, episode:  160\n",
      "frames: 334000, reward: -7.300000, total_loss: 1.737044, forward_loss: 0.000000, inverse_pred_loss: 1.736669, Q_loss: 0.000375, intrinsic_rewards: 0.000002, epsilon: 0.011243, episode:  161\n",
      "frames: 335000, reward: -7.300000, total_loss: 1.772624, forward_loss: 0.000000, inverse_pred_loss: 1.770969, Q_loss: 0.001654, intrinsic_rewards: 0.000001, epsilon: 0.011219, episode:  161\n",
      "frames: 336000, reward: -7.300000, total_loss: 1.748935, forward_loss: 0.000000, inverse_pred_loss: 1.748404, Q_loss: 0.000531, intrinsic_rewards: 0.000001, epsilon: 0.011194, episode:  161\n",
      "frames: 337000, reward: -6.700000, total_loss: 1.720815, forward_loss: 0.000000, inverse_pred_loss: 1.719797, Q_loss: 0.001018, intrinsic_rewards: 0.000001, epsilon: 0.011171, episode:  162\n",
      "frames: 338000, reward: -6.700000, total_loss: 1.774901, forward_loss: 0.000000, inverse_pred_loss: 1.774090, Q_loss: 0.000811, intrinsic_rewards: 0.000001, epsilon: 0.011148, episode:  162\n",
      "frames: 339000, reward: -6.700000, total_loss: 1.785893, forward_loss: 0.000000, inverse_pred_loss: 1.785107, Q_loss: 0.000785, intrinsic_rewards: 0.000002, epsilon: 0.011125, episode:  162\n",
      "frames: 340000, reward: -4.700000, total_loss: 1.760615, forward_loss: 0.000000, inverse_pred_loss: 1.759638, Q_loss: 0.000977, intrinsic_rewards: 0.000001, epsilon: 0.011103, episode:  163\n",
      "frames: 341000, reward: -4.700000, total_loss: 1.773638, forward_loss: 0.000000, inverse_pred_loss: 1.773094, Q_loss: 0.000544, intrinsic_rewards: 0.000001, epsilon: 0.011081, episode:  163\n",
      "frames: 342000, reward: -4.700000, total_loss: 1.815128, forward_loss: 0.000000, inverse_pred_loss: 1.812313, Q_loss: 0.002815, intrinsic_rewards: 0.000003, epsilon: 0.011059, episode:  163\n",
      "frames: 343000, reward: -4.600000, total_loss: 1.815248, forward_loss: 0.000000, inverse_pred_loss: 1.814035, Q_loss: 0.001213, intrinsic_rewards: 0.000002, epsilon: 0.011038, episode:  164\n",
      "frames: 344000, reward: -4.600000, total_loss: 1.762959, forward_loss: 0.000000, inverse_pred_loss: 1.762387, Q_loss: 0.000572, intrinsic_rewards: 0.000000, epsilon: 0.011018, episode:  164\n",
      "frames: 345000, reward: -4.600000, total_loss: 1.777173, forward_loss: 0.000000, inverse_pred_loss: 1.776151, Q_loss: 0.001023, intrinsic_rewards: 0.000001, epsilon: 0.010998, episode:  164\n",
      "frames: 346000, reward: -4.600000, total_loss: 1.753680, forward_loss: 0.000000, inverse_pred_loss: 1.752406, Q_loss: 0.001274, intrinsic_rewards: 0.000002, epsilon: 0.010978, episode:  164\n",
      "frames: 347000, reward: -3.300000, total_loss: 1.752130, forward_loss: 0.000000, inverse_pred_loss: 1.751047, Q_loss: 0.001083, intrinsic_rewards: 0.000011, epsilon: 0.010959, episode:  165\n",
      "frames: 348000, reward: -3.300000, total_loss: 1.763087, forward_loss: 0.000000, inverse_pred_loss: 1.760687, Q_loss: 0.002400, intrinsic_rewards: 0.000003, epsilon: 0.010940, episode:  165\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 349000, reward: -3.300000, total_loss: 1.741904, forward_loss: 0.000000, inverse_pred_loss: 1.741125, Q_loss: 0.000779, intrinsic_rewards: 0.000002, epsilon: 0.010921, episode:  165\n",
      "frames: 350000, reward: -3.300000, total_loss: 1.831304, forward_loss: 0.000000, inverse_pred_loss: 1.830678, Q_loss: 0.000626, intrinsic_rewards: 0.000001, epsilon: 0.010903, episode:  165\n",
      "frames: 351000, reward: -2.100000, total_loss: 1.761817, forward_loss: 0.000000, inverse_pred_loss: 1.760885, Q_loss: 0.000931, intrinsic_rewards: 0.000001, epsilon: 0.010885, episode:  166\n",
      "frames: 352000, reward: -2.100000, total_loss: 1.765632, forward_loss: 0.000000, inverse_pred_loss: 1.765103, Q_loss: 0.000530, intrinsic_rewards: 0.000001, epsilon: 0.010867, episode:  166\n",
      "frames: 353000, reward: -2.100000, total_loss: 1.752459, forward_loss: 0.000000, inverse_pred_loss: 1.750868, Q_loss: 0.001591, intrinsic_rewards: 0.000002, epsilon: 0.010850, episode:  166\n",
      "frames: 354000, reward: -1.200000, total_loss: 1.770705, forward_loss: 0.000000, inverse_pred_loss: 1.770149, Q_loss: 0.000556, intrinsic_rewards: 0.000003, epsilon: 0.010833, episode:  167\n",
      "frames: 355000, reward: -1.200000, total_loss: 1.770877, forward_loss: 0.000000, inverse_pred_loss: 1.769632, Q_loss: 0.001245, intrinsic_rewards: 0.000001, epsilon: 0.010817, episode:  167\n",
      "frames: 356000, reward: -1.200000, total_loss: 1.734022, forward_loss: 0.000000, inverse_pred_loss: 1.733198, Q_loss: 0.000823, intrinsic_rewards: 0.000001, epsilon: 0.010801, episode:  167\n",
      "frames: 357000, reward: -1.200000, total_loss: 1.810954, forward_loss: 0.000000, inverse_pred_loss: 1.810533, Q_loss: 0.000421, intrinsic_rewards: 0.000002, epsilon: 0.010785, episode:  168\n",
      "frames: 358000, reward: -1.200000, total_loss: 1.806715, forward_loss: 0.000000, inverse_pred_loss: 1.806086, Q_loss: 0.000629, intrinsic_rewards: 0.000002, epsilon: 0.010769, episode:  168\n",
      "frames: 359000, reward: -1.200000, total_loss: 1.774635, forward_loss: 0.000000, inverse_pred_loss: 1.773794, Q_loss: 0.000841, intrinsic_rewards: 0.000001, epsilon: 0.010754, episode:  168\n",
      "frames: 360000, reward: -1.200000, total_loss: 1.734854, forward_loss: 0.000000, inverse_pred_loss: 1.731956, Q_loss: 0.002898, intrinsic_rewards: 0.000004, epsilon: 0.010739, episode:  168\n",
      "frames: 361000, reward: -0.400000, total_loss: 1.774680, forward_loss: 0.000000, inverse_pred_loss: 1.774018, Q_loss: 0.000662, intrinsic_rewards: 0.000003, epsilon: 0.010724, episode:  169\n",
      "frames: 362000, reward: -0.400000, total_loss: 1.732611, forward_loss: 0.000000, inverse_pred_loss: 1.731354, Q_loss: 0.001258, intrinsic_rewards: 0.000004, epsilon: 0.010710, episode:  169\n",
      "frames: 363000, reward: -0.400000, total_loss: 1.784878, forward_loss: 0.000000, inverse_pred_loss: 1.784350, Q_loss: 0.000528, intrinsic_rewards: 0.000003, epsilon: 0.010696, episode:  169\n",
      "frames: 364000, reward: 0.500000, total_loss: 1.751957, forward_loss: 0.000000, inverse_pred_loss: 1.751553, Q_loss: 0.000404, intrinsic_rewards: 0.000002, epsilon: 0.010682, episode:  170\n",
      "frames: 365000, reward: 0.500000, total_loss: 1.781619, forward_loss: 0.000000, inverse_pred_loss: 1.780118, Q_loss: 0.001501, intrinsic_rewards: 0.000003, epsilon: 0.010669, episode:  170\n",
      "frames: 366000, reward: 0.500000, total_loss: 1.730198, forward_loss: 0.000000, inverse_pred_loss: 1.729500, Q_loss: 0.000698, intrinsic_rewards: 0.000005, epsilon: 0.010656, episode:  170\n",
      "frames: 367000, reward: 0.500000, total_loss: 1.751094, forward_loss: 0.000000, inverse_pred_loss: 1.750711, Q_loss: 0.000383, intrinsic_rewards: 0.000003, epsilon: 0.010643, episode:  170\n",
      "frames: 368000, reward: 1.400000, total_loss: 1.779192, forward_loss: 0.000000, inverse_pred_loss: 1.777711, Q_loss: 0.001481, intrinsic_rewards: 0.000006, epsilon: 0.010630, episode:  171\n",
      "frames: 369000, reward: 1.400000, total_loss: 1.765466, forward_loss: 0.000000, inverse_pred_loss: 1.764669, Q_loss: 0.000797, intrinsic_rewards: 0.000016, epsilon: 0.010617, episode:  171\n",
      "frames: 370000, reward: 1.400000, total_loss: 1.734819, forward_loss: 0.000000, inverse_pred_loss: 1.734248, Q_loss: 0.000571, intrinsic_rewards: 0.000001, epsilon: 0.010605, episode:  171\n",
      "frames: 371000, reward: 1.400000, total_loss: 1.747179, forward_loss: 0.000000, inverse_pred_loss: 1.745773, Q_loss: 0.001406, intrinsic_rewards: 0.000002, epsilon: 0.010593, episode:  172\n",
      "frames: 372000, reward: 1.400000, total_loss: 1.768654, forward_loss: 0.000000, inverse_pred_loss: 1.767936, Q_loss: 0.000717, intrinsic_rewards: 0.000001, epsilon: 0.010581, episode:  172\n",
      "frames: 373000, reward: 1.400000, total_loss: 1.788209, forward_loss: 0.000000, inverse_pred_loss: 1.787781, Q_loss: 0.000428, intrinsic_rewards: 0.000001, epsilon: 0.010570, episode:  172\n",
      "frames: 374000, reward: 1.400000, total_loss: 1.811902, forward_loss: 0.000000, inverse_pred_loss: 1.810806, Q_loss: 0.001097, intrinsic_rewards: 0.000002, epsilon: 0.010559, episode:  172\n",
      "frames: 375000, reward: 0.400000, total_loss: 1.782580, forward_loss: 0.000000, inverse_pred_loss: 1.781952, Q_loss: 0.000628, intrinsic_rewards: 0.000004, epsilon: 0.010548, episode:  173\n",
      "frames: 376000, reward: 0.400000, total_loss: 1.806812, forward_loss: 0.000000, inverse_pred_loss: 1.806096, Q_loss: 0.000717, intrinsic_rewards: 0.000003, epsilon: 0.010537, episode:  173\n",
      "frames: 377000, reward: 3.200000, total_loss: 1.744504, forward_loss: 0.000000, inverse_pred_loss: 1.743652, Q_loss: 0.000852, intrinsic_rewards: 0.000010, epsilon: 0.010526, episode:  174\n",
      "frames: 378000, reward: 3.200000, total_loss: 1.769216, forward_loss: 0.000000, inverse_pred_loss: 1.768244, Q_loss: 0.000972, intrinsic_rewards: 0.000001, epsilon: 0.010516, episode:  174\n",
      "frames: 379000, reward: 3.200000, total_loss: 1.744781, forward_loss: 0.000000, inverse_pred_loss: 1.744210, Q_loss: 0.000571, intrinsic_rewards: 0.000002, epsilon: 0.010505, episode:  174\n",
      "frames: 380000, reward: 3.800000, total_loss: 1.795419, forward_loss: 0.000000, inverse_pred_loss: 1.793470, Q_loss: 0.001949, intrinsic_rewards: 0.000001, epsilon: 0.010495, episode:  175\n",
      "frames: 381000, reward: 3.800000, total_loss: 1.756353, forward_loss: 0.000000, inverse_pred_loss: 1.754977, Q_loss: 0.001376, intrinsic_rewards: 0.000007, epsilon: 0.010486, episode:  175\n",
      "frames: 382000, reward: 4.700000, total_loss: 1.779919, forward_loss: 0.000000, inverse_pred_loss: 1.778991, Q_loss: 0.000928, intrinsic_rewards: 0.000001, epsilon: 0.010476, episode:  176\n",
      "frames: 383000, reward: 4.700000, total_loss: 1.803364, forward_loss: 0.000000, inverse_pred_loss: 1.802756, Q_loss: 0.000607, intrinsic_rewards: 0.000000, epsilon: 0.010467, episode:  176\n",
      "frames: 384000, reward: 4.700000, total_loss: 1.755363, forward_loss: 0.000000, inverse_pred_loss: 1.754874, Q_loss: 0.000489, intrinsic_rewards: 0.000000, epsilon: 0.010457, episode:  176\n",
      "frames: 385000, reward: 5.900000, total_loss: 1.766151, forward_loss: 0.000000, inverse_pred_loss: 1.763940, Q_loss: 0.002211, intrinsic_rewards: 0.000002, epsilon: 0.010448, episode:  177\n",
      "frames: 386000, reward: 5.900000, total_loss: 1.717254, forward_loss: 0.000000, inverse_pred_loss: 1.716479, Q_loss: 0.000775, intrinsic_rewards: 0.000003, epsilon: 0.010439, episode:  177\n",
      "frames: 387000, reward: 5.900000, total_loss: 1.725321, forward_loss: 0.000000, inverse_pred_loss: 1.724396, Q_loss: 0.000925, intrinsic_rewards: 0.000005, epsilon: 0.010431, episode:  177\n",
      "frames: 388000, reward: 5.900000, total_loss: 1.753098, forward_loss: 0.000000, inverse_pred_loss: 1.752690, Q_loss: 0.000408, intrinsic_rewards: 0.000001, epsilon: 0.010422, episode:  177\n",
      "frames: 389000, reward: 6.400000, total_loss: 1.763155, forward_loss: 0.000000, inverse_pred_loss: 1.762535, Q_loss: 0.000620, intrinsic_rewards: 0.000001, epsilon: 0.010414, episode:  178\n",
      "frames: 390000, reward: 6.400000, total_loss: 1.747739, forward_loss: 0.000000, inverse_pred_loss: 1.747148, Q_loss: 0.000591, intrinsic_rewards: 0.000001, epsilon: 0.010406, episode:  178\n",
      "frames: 391000, reward: 6.400000, total_loss: 1.760943, forward_loss: 0.000000, inverse_pred_loss: 1.760579, Q_loss: 0.000364, intrinsic_rewards: 0.000001, epsilon: 0.010398, episode:  178\n",
      "frames: 392000, reward: 6.200000, total_loss: 1.770609, forward_loss: 0.000000, inverse_pred_loss: 1.769844, Q_loss: 0.000766, intrinsic_rewards: 0.000001, epsilon: 0.010390, episode:  179\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 393000, reward: 6.200000, total_loss: 1.779951, forward_loss: 0.000000, inverse_pred_loss: 1.779468, Q_loss: 0.000483, intrinsic_rewards: 0.000003, epsilon: 0.010382, episode:  179\n",
      "frames: 394000, reward: 7.200000, total_loss: 1.839373, forward_loss: 0.000000, inverse_pred_loss: 1.837721, Q_loss: 0.001652, intrinsic_rewards: 0.000006, epsilon: 0.010374, episode:  180\n",
      "frames: 395000, reward: 7.200000, total_loss: 1.797602, forward_loss: 0.000000, inverse_pred_loss: 1.796190, Q_loss: 0.001412, intrinsic_rewards: 0.000001, epsilon: 0.010367, episode:  180\n",
      "frames: 396000, reward: 7.200000, total_loss: 1.780483, forward_loss: 0.000000, inverse_pred_loss: 1.780157, Q_loss: 0.000326, intrinsic_rewards: 0.000001, epsilon: 0.010360, episode:  180\n",
      "frames: 397000, reward: 8.800000, total_loss: 1.794863, forward_loss: 0.000000, inverse_pred_loss: 1.794314, Q_loss: 0.000550, intrinsic_rewards: 0.000001, epsilon: 0.010353, episode:  181\n",
      "frames: 398000, reward: 8.800000, total_loss: 1.779760, forward_loss: 0.000000, inverse_pred_loss: 1.779138, Q_loss: 0.000622, intrinsic_rewards: 0.000003, epsilon: 0.010346, episode:  181\n",
      "frames: 399000, reward: 10.000000, total_loss: 1.765820, forward_loss: 0.000000, inverse_pred_loss: 1.765052, Q_loss: 0.000768, intrinsic_rewards: 0.000001, epsilon: 0.010339, episode:  182\n",
      "frames: 400000, reward: 10.000000, total_loss: 1.752768, forward_loss: 0.000000, inverse_pred_loss: 1.752058, Q_loss: 0.000709, intrinsic_rewards: 0.000001, epsilon: 0.010332, episode:  182\n",
      "frames: 401000, reward: 11.500000, total_loss: 1.787645, forward_loss: 0.000000, inverse_pred_loss: 1.787059, Q_loss: 0.000585, intrinsic_rewards: 0.000002, epsilon: 0.010326, episode:  183\n",
      "frames: 402000, reward: 11.500000, total_loss: 1.775019, forward_loss: 0.000000, inverse_pred_loss: 1.774601, Q_loss: 0.000419, intrinsic_rewards: 0.000001, epsilon: 0.010319, episode:  183\n",
      "frames: 403000, reward: 11.200000, total_loss: 1.757561, forward_loss: 0.000000, inverse_pred_loss: 1.757296, Q_loss: 0.000265, intrinsic_rewards: 0.000000, epsilon: 0.010313, episode:  184\n",
      "frames: 404000, reward: 11.200000, total_loss: 1.817814, forward_loss: 0.000000, inverse_pred_loss: 1.817235, Q_loss: 0.000579, intrinsic_rewards: 0.000000, epsilon: 0.010307, episode:  184\n",
      "frames: 405000, reward: 11.900000, total_loss: 1.783453, forward_loss: 0.000000, inverse_pred_loss: 1.783046, Q_loss: 0.000408, intrinsic_rewards: 0.000000, epsilon: 0.010301, episode:  185\n",
      "frames: 406000, reward: 11.900000, total_loss: 1.798942, forward_loss: 0.000000, inverse_pred_loss: 1.798611, Q_loss: 0.000331, intrinsic_rewards: 0.000000, epsilon: 0.010295, episode:  185\n",
      "frames: 407000, reward: 12.100000, total_loss: 1.741110, forward_loss: 0.000000, inverse_pred_loss: 1.740630, Q_loss: 0.000480, intrinsic_rewards: 0.000002, epsilon: 0.010289, episode:  186\n",
      "frames: 408000, reward: 12.100000, total_loss: 1.793449, forward_loss: 0.000000, inverse_pred_loss: 1.793005, Q_loss: 0.000444, intrinsic_rewards: 0.000002, epsilon: 0.010283, episode:  186\n",
      "frames: 409000, reward: 12.600000, total_loss: 1.801640, forward_loss: 0.000000, inverse_pred_loss: 1.801126, Q_loss: 0.000515, intrinsic_rewards: 0.000002, epsilon: 0.010277, episode:  187\n",
      "frames: 410000, reward: 12.600000, total_loss: 1.765220, forward_loss: 0.000000, inverse_pred_loss: 1.764668, Q_loss: 0.000552, intrinsic_rewards: 0.000002, epsilon: 0.010272, episode:  187\n",
      "frames: 411000, reward: 14.100000, total_loss: 1.769283, forward_loss: 0.000000, inverse_pred_loss: 1.769066, Q_loss: 0.000217, intrinsic_rewards: 0.000000, epsilon: 0.010267, episode:  188\n",
      "frames: 412000, reward: 14.100000, total_loss: 1.760714, forward_loss: 0.000000, inverse_pred_loss: 1.760553, Q_loss: 0.000161, intrinsic_rewards: 0.000001, epsilon: 0.010261, episode:  188\n",
      "frames: 413000, reward: 15.900000, total_loss: 1.744913, forward_loss: 0.000000, inverse_pred_loss: 1.744564, Q_loss: 0.000349, intrinsic_rewards: 0.000001, epsilon: 0.010256, episode:  189\n",
      "frames: 414000, reward: 15.900000, total_loss: 1.774827, forward_loss: 0.000000, inverse_pred_loss: 1.774500, Q_loss: 0.000328, intrinsic_rewards: 0.000000, epsilon: 0.010251, episode:  189\n",
      "frames: 415000, reward: 15.900000, total_loss: 1.776677, forward_loss: 0.000000, inverse_pred_loss: 1.776169, Q_loss: 0.000508, intrinsic_rewards: 0.000001, epsilon: 0.010246, episode:  190\n",
      "frames: 416000, reward: 15.900000, total_loss: 1.727854, forward_loss: 0.000000, inverse_pred_loss: 1.727470, Q_loss: 0.000384, intrinsic_rewards: 0.000001, epsilon: 0.010241, episode:  190\n",
      "frames: 417000, reward: 16.000000, total_loss: 1.797288, forward_loss: 0.000000, inverse_pred_loss: 1.796950, Q_loss: 0.000339, intrinsic_rewards: 0.000000, epsilon: 0.010236, episode:  191\n",
      "frames: 418000, reward: 16.000000, total_loss: 1.785972, forward_loss: 0.000000, inverse_pred_loss: 1.785598, Q_loss: 0.000373, intrinsic_rewards: 0.000001, epsilon: 0.010232, episode:  191\n",
      "frames: 419000, reward: 16.000000, total_loss: 1.719671, forward_loss: 0.000000, inverse_pred_loss: 1.719193, Q_loss: 0.000477, intrinsic_rewards: 0.000001, epsilon: 0.010227, episode:  192\n",
      "frames: 420000, reward: 16.000000, total_loss: 1.800754, forward_loss: 0.000000, inverse_pred_loss: 1.800426, Q_loss: 0.000328, intrinsic_rewards: 0.000000, epsilon: 0.010223, episode:  192\n",
      "frames: 421000, reward: 16.000000, total_loss: 1.789990, forward_loss: 0.000000, inverse_pred_loss: 1.789217, Q_loss: 0.000774, intrinsic_rewards: 0.000001, epsilon: 0.010218, episode:  192\n",
      "frames: 422000, reward: 15.200000, total_loss: 1.796261, forward_loss: 0.000000, inverse_pred_loss: 1.795864, Q_loss: 0.000396, intrinsic_rewards: 0.000001, epsilon: 0.010214, episode:  193\n",
      "frames: 423000, reward: 15.200000, total_loss: 1.797183, forward_loss: 0.000000, inverse_pred_loss: 1.795922, Q_loss: 0.001261, intrinsic_rewards: 0.000000, epsilon: 0.010210, episode:  193\n",
      "frames: 424000, reward: 15.200000, total_loss: 1.772640, forward_loss: 0.000000, inverse_pred_loss: 1.772295, Q_loss: 0.000345, intrinsic_rewards: 0.000001, epsilon: 0.010206, episode:  194\n",
      "frames: 425000, reward: 15.200000, total_loss: 1.756631, forward_loss: 0.000000, inverse_pred_loss: 1.756053, Q_loss: 0.000577, intrinsic_rewards: 0.000000, epsilon: 0.010201, episode:  194\n",
      "frames: 426000, reward: 15.300000, total_loss: 1.767006, forward_loss: 0.000000, inverse_pred_loss: 1.766490, Q_loss: 0.000516, intrinsic_rewards: 0.000000, epsilon: 0.010197, episode:  195\n",
      "frames: 427000, reward: 15.300000, total_loss: 1.823214, forward_loss: 0.000000, inverse_pred_loss: 1.822144, Q_loss: 0.001070, intrinsic_rewards: 0.000000, epsilon: 0.010194, episode:  195\n",
      "frames: 428000, reward: 15.300000, total_loss: 1.795112, forward_loss: 0.000000, inverse_pred_loss: 1.794319, Q_loss: 0.000793, intrinsic_rewards: 0.000000, epsilon: 0.010190, episode:  195\n",
      "frames: 429000, reward: 14.600000, total_loss: 1.804035, forward_loss: 0.000000, inverse_pred_loss: 1.803742, Q_loss: 0.000293, intrinsic_rewards: 0.000001, epsilon: 0.010186, episode:  196\n",
      "frames: 430000, reward: 14.600000, total_loss: 1.795338, forward_loss: 0.000000, inverse_pred_loss: 1.795018, Q_loss: 0.000320, intrinsic_rewards: 0.000000, epsilon: 0.010182, episode:  196\n",
      "frames: 431000, reward: 14.600000, total_loss: 1.772097, forward_loss: 0.000000, inverse_pred_loss: 1.771416, Q_loss: 0.000681, intrinsic_rewards: 0.000000, epsilon: 0.010179, episode:  197\n",
      "frames: 432000, reward: 14.600000, total_loss: 1.785869, forward_loss: 0.000000, inverse_pred_loss: 1.785194, Q_loss: 0.000675, intrinsic_rewards: 0.000000, epsilon: 0.010175, episode:  197\n",
      "frames: 433000, reward: 14.500000, total_loss: 1.763751, forward_loss: 0.000000, inverse_pred_loss: 1.763560, Q_loss: 0.000191, intrinsic_rewards: 0.000001, epsilon: 0.010172, episode:  198\n",
      "frames: 434000, reward: 14.500000, total_loss: 1.797815, forward_loss: 0.000000, inverse_pred_loss: 1.797314, Q_loss: 0.000501, intrinsic_rewards: 0.000002, epsilon: 0.010168, episode:  198\n",
      "frames: 435000, reward: 14.600000, total_loss: 1.802611, forward_loss: 0.000000, inverse_pred_loss: 1.800828, Q_loss: 0.001784, intrinsic_rewards: 0.000000, epsilon: 0.010165, episode:  199\n",
      "frames: 436000, reward: 14.600000, total_loss: 1.753160, forward_loss: 0.000000, inverse_pred_loss: 1.752589, Q_loss: 0.000571, intrinsic_rewards: 0.000002, epsilon: 0.010162, episode:  199\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 437000, reward: 14.400000, total_loss: 1.784516, forward_loss: 0.000000, inverse_pred_loss: 1.783893, Q_loss: 0.000622, intrinsic_rewards: 0.000001, epsilon: 0.010158, episode:  200\n",
      "frames: 438000, reward: 14.400000, total_loss: 1.738479, forward_loss: 0.000000, inverse_pred_loss: 1.738197, Q_loss: 0.000281, intrinsic_rewards: 0.000002, epsilon: 0.010155, episode:  200\n",
      "frames: 439000, reward: 14.500000, total_loss: 1.751054, forward_loss: 0.000000, inverse_pred_loss: 1.750443, Q_loss: 0.000611, intrinsic_rewards: 0.000001, epsilon: 0.010152, episode:  201\n",
      "frames: 440000, reward: 14.500000, total_loss: 1.756575, forward_loss: 0.000000, inverse_pred_loss: 1.756406, Q_loss: 0.000169, intrinsic_rewards: 0.000000, epsilon: 0.010149, episode:  201\n",
      "frames: 441000, reward: 14.200000, total_loss: 1.740761, forward_loss: 0.000000, inverse_pred_loss: 1.740248, Q_loss: 0.000513, intrinsic_rewards: 0.000000, epsilon: 0.010146, episode:  202\n",
      "frames: 442000, reward: 14.200000, total_loss: 1.804884, forward_loss: 0.000000, inverse_pred_loss: 1.803413, Q_loss: 0.001471, intrinsic_rewards: 0.000001, epsilon: 0.010143, episode:  202\n",
      "frames: 443000, reward: 14.700000, total_loss: 1.804532, forward_loss: 0.000000, inverse_pred_loss: 1.804155, Q_loss: 0.000377, intrinsic_rewards: 0.000001, epsilon: 0.010141, episode:  203\n",
      "frames: 444000, reward: 14.700000, total_loss: 1.783138, forward_loss: 0.000000, inverse_pred_loss: 1.782531, Q_loss: 0.000608, intrinsic_rewards: 0.000000, epsilon: 0.010138, episode:  203\n",
      "frames: 445000, reward: 14.700000, total_loss: 1.775879, forward_loss: 0.000000, inverse_pred_loss: 1.774985, Q_loss: 0.000894, intrinsic_rewards: 0.000000, epsilon: 0.010135, episode:  203\n",
      "frames: 446000, reward: 14.300000, total_loss: 1.810022, forward_loss: 0.000000, inverse_pred_loss: 1.809423, Q_loss: 0.000600, intrinsic_rewards: 0.000001, epsilon: 0.010132, episode:  204\n",
      "frames: 447000, reward: 14.300000, total_loss: 1.806097, forward_loss: 0.000000, inverse_pred_loss: 1.805430, Q_loss: 0.000667, intrinsic_rewards: 0.000001, epsilon: 0.010130, episode:  204\n",
      "frames: 448000, reward: 14.400000, total_loss: 1.811646, forward_loss: 0.000000, inverse_pred_loss: 1.811391, Q_loss: 0.000255, intrinsic_rewards: 0.000000, epsilon: 0.010127, episode:  205\n",
      "frames: 449000, reward: 14.400000, total_loss: 1.777508, forward_loss: 0.000000, inverse_pred_loss: 1.777198, Q_loss: 0.000310, intrinsic_rewards: 0.000000, epsilon: 0.010125, episode:  205\n",
      "frames: 450000, reward: 15.400000, total_loss: 1.723777, forward_loss: 0.000000, inverse_pred_loss: 1.723078, Q_loss: 0.000699, intrinsic_rewards: 0.000001, epsilon: 0.010122, episode:  206\n",
      "frames: 451000, reward: 15.400000, total_loss: 1.802345, forward_loss: 0.000000, inverse_pred_loss: 1.801851, Q_loss: 0.000494, intrinsic_rewards: 0.000001, epsilon: 0.010120, episode:  206\n",
      "frames: 452000, reward: 15.600000, total_loss: 1.825505, forward_loss: 0.000000, inverse_pred_loss: 1.825027, Q_loss: 0.000477, intrinsic_rewards: 0.000001, epsilon: 0.010117, episode:  207\n",
      "frames: 453000, reward: 15.600000, total_loss: 1.754430, forward_loss: 0.000000, inverse_pred_loss: 1.754009, Q_loss: 0.000421, intrinsic_rewards: 0.000001, epsilon: 0.010115, episode:  207\n",
      "frames: 454000, reward: 15.400000, total_loss: 1.805638, forward_loss: 0.000000, inverse_pred_loss: 1.805406, Q_loss: 0.000232, intrinsic_rewards: 0.000000, epsilon: 0.010113, episode:  208\n",
      "frames: 455000, reward: 15.400000, total_loss: 1.801467, forward_loss: 0.000000, inverse_pred_loss: 1.801278, Q_loss: 0.000188, intrinsic_rewards: 0.000003, epsilon: 0.010111, episode:  208\n",
      "frames: 456000, reward: 15.500000, total_loss: 1.807306, forward_loss: 0.000000, inverse_pred_loss: 1.807081, Q_loss: 0.000225, intrinsic_rewards: 0.000001, epsilon: 0.010108, episode:  209\n",
      "frames: 457000, reward: 15.500000, total_loss: 1.798111, forward_loss: 0.000000, inverse_pred_loss: 1.797275, Q_loss: 0.000836, intrinsic_rewards: 0.000002, epsilon: 0.010106, episode:  209\n",
      "frames: 458000, reward: 15.600000, total_loss: 1.821397, forward_loss: 0.000000, inverse_pred_loss: 1.820486, Q_loss: 0.000911, intrinsic_rewards: 0.000001, epsilon: 0.010104, episode:  210\n",
      "frames: 459000, reward: 15.600000, total_loss: 1.759298, forward_loss: 0.000000, inverse_pred_loss: 1.759147, Q_loss: 0.000150, intrinsic_rewards: 0.000002, epsilon: 0.010102, episode:  210\n",
      "frames: 460000, reward: 15.400000, total_loss: 1.774662, forward_loss: 0.000000, inverse_pred_loss: 1.773318, Q_loss: 0.001345, intrinsic_rewards: 0.000002, epsilon: 0.010100, episode:  211\n",
      "frames: 461000, reward: 15.400000, total_loss: 1.764506, forward_loss: 0.000000, inverse_pred_loss: 1.764250, Q_loss: 0.000256, intrinsic_rewards: 0.000001, epsilon: 0.010098, episode:  211\n",
      "frames: 462000, reward: 15.900000, total_loss: 1.814675, forward_loss: 0.000000, inverse_pred_loss: 1.814376, Q_loss: 0.000299, intrinsic_rewards: 0.000004, epsilon: 0.010096, episode:  212\n",
      "frames: 463000, reward: 15.900000, total_loss: 1.811217, forward_loss: 0.000000, inverse_pred_loss: 1.810605, Q_loss: 0.000612, intrinsic_rewards: 0.000001, epsilon: 0.010094, episode:  212\n",
      "frames: 464000, reward: 16.000000, total_loss: 1.815695, forward_loss: 0.000000, inverse_pred_loss: 1.815331, Q_loss: 0.000364, intrinsic_rewards: 0.000001, epsilon: 0.010092, episode:  213\n",
      "frames: 465000, reward: 16.000000, total_loss: 1.848245, forward_loss: 0.000000, inverse_pred_loss: 1.847830, Q_loss: 0.000416, intrinsic_rewards: 0.000002, epsilon: 0.010091, episode:  213\n",
      "frames: 466000, reward: 16.200000, total_loss: 1.833389, forward_loss: 0.000000, inverse_pred_loss: 1.832895, Q_loss: 0.000494, intrinsic_rewards: 0.000002, epsilon: 0.010089, episode:  214\n",
      "frames: 467000, reward: 16.200000, total_loss: 1.811233, forward_loss: 0.000000, inverse_pred_loss: 1.810962, Q_loss: 0.000271, intrinsic_rewards: 0.000001, epsilon: 0.010087, episode:  214\n",
      "frames: 468000, reward: 16.200000, total_loss: 1.815446, forward_loss: 0.000000, inverse_pred_loss: 1.815049, Q_loss: 0.000397, intrinsic_rewards: 0.000001, epsilon: 0.010085, episode:  214\n",
      "frames: 469000, reward: 16.200000, total_loss: 1.785223, forward_loss: 0.000000, inverse_pred_loss: 1.784754, Q_loss: 0.000469, intrinsic_rewards: 0.000000, epsilon: 0.010084, episode:  215\n",
      "frames: 470000, reward: 16.200000, total_loss: 1.759268, forward_loss: 0.000000, inverse_pred_loss: 1.759022, Q_loss: 0.000246, intrinsic_rewards: 0.000002, epsilon: 0.010082, episode:  215\n",
      "frames: 471000, reward: 15.700000, total_loss: 1.783380, forward_loss: 0.000000, inverse_pred_loss: 1.783106, Q_loss: 0.000274, intrinsic_rewards: 0.000001, epsilon: 0.010080, episode:  216\n",
      "frames: 472000, reward: 15.700000, total_loss: 1.761738, forward_loss: 0.000000, inverse_pred_loss: 1.761330, Q_loss: 0.000408, intrinsic_rewards: 0.000000, epsilon: 0.010079, episode:  216\n",
      "frames: 473000, reward: 15.600000, total_loss: 1.732104, forward_loss: 0.000000, inverse_pred_loss: 1.731595, Q_loss: 0.000510, intrinsic_rewards: 0.000001, epsilon: 0.010077, episode:  217\n",
      "frames: 474000, reward: 15.600000, total_loss: 1.767490, forward_loss: 0.000000, inverse_pred_loss: 1.766854, Q_loss: 0.000636, intrinsic_rewards: 0.000001, epsilon: 0.010076, episode:  217\n",
      "frames: 475000, reward: 15.800000, total_loss: 1.780457, forward_loss: 0.000000, inverse_pred_loss: 1.780171, Q_loss: 0.000286, intrinsic_rewards: 0.000001, epsilon: 0.010074, episode:  218\n",
      "frames: 476000, reward: 15.800000, total_loss: 1.830975, forward_loss: 0.000000, inverse_pred_loss: 1.830646, Q_loss: 0.000329, intrinsic_rewards: 0.000001, epsilon: 0.010073, episode:  218\n",
      "frames: 477000, reward: 15.600000, total_loss: 1.779702, forward_loss: 0.000000, inverse_pred_loss: 1.779243, Q_loss: 0.000459, intrinsic_rewards: 0.000001, epsilon: 0.010071, episode:  219\n",
      "frames: 478000, reward: 15.600000, total_loss: 1.831989, forward_loss: 0.000000, inverse_pred_loss: 1.831681, Q_loss: 0.000308, intrinsic_rewards: 0.000000, epsilon: 0.010070, episode:  219\n",
      "frames: 479000, reward: 15.700000, total_loss: 1.821099, forward_loss: 0.000000, inverse_pred_loss: 1.820771, Q_loss: 0.000329, intrinsic_rewards: 0.000001, epsilon: 0.010068, episode:  220\n",
      "frames: 480000, reward: 15.700000, total_loss: 1.777710, forward_loss: 0.000000, inverse_pred_loss: 1.777441, Q_loss: 0.000269, intrinsic_rewards: 0.000001, epsilon: 0.010067, episode:  220\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 481000, reward: 16.200000, total_loss: 1.837844, forward_loss: 0.000000, inverse_pred_loss: 1.837255, Q_loss: 0.000589, intrinsic_rewards: 0.000001, epsilon: 0.010066, episode:  221\n",
      "frames: 482000, reward: 16.200000, total_loss: 1.829274, forward_loss: 0.000000, inverse_pred_loss: 1.829071, Q_loss: 0.000203, intrinsic_rewards: 0.000000, epsilon: 0.010064, episode:  221\n",
      "frames: 483000, reward: 15.700000, total_loss: 1.781961, forward_loss: 0.000000, inverse_pred_loss: 1.781581, Q_loss: 0.000380, intrinsic_rewards: 0.000001, epsilon: 0.010063, episode:  222\n",
      "frames: 484000, reward: 15.700000, total_loss: 1.838534, forward_loss: 0.000000, inverse_pred_loss: 1.838321, Q_loss: 0.000213, intrinsic_rewards: 0.000000, epsilon: 0.010062, episode:  222\n",
      "frames: 485000, reward: 16.000000, total_loss: 1.755851, forward_loss: 0.000000, inverse_pred_loss: 1.755481, Q_loss: 0.000369, intrinsic_rewards: 0.000001, epsilon: 0.010061, episode:  223\n",
      "frames: 486000, reward: 16.000000, total_loss: 1.859152, forward_loss: 0.000000, inverse_pred_loss: 1.858899, Q_loss: 0.000253, intrinsic_rewards: 0.000001, epsilon: 0.010059, episode:  223\n",
      "frames: 487000, reward: 16.600000, total_loss: 1.797475, forward_loss: 0.000000, inverse_pred_loss: 1.797120, Q_loss: 0.000356, intrinsic_rewards: 0.000001, epsilon: 0.010058, episode:  224\n",
      "frames: 488000, reward: 16.600000, total_loss: 1.795663, forward_loss: 0.000000, inverse_pred_loss: 1.795390, Q_loss: 0.000273, intrinsic_rewards: 0.000000, epsilon: 0.010057, episode:  224\n",
      "frames: 489000, reward: 16.600000, total_loss: 1.769554, forward_loss: 0.000000, inverse_pred_loss: 1.769187, Q_loss: 0.000368, intrinsic_rewards: 0.000002, epsilon: 0.010056, episode:  225\n",
      "frames: 490000, reward: 16.600000, total_loss: 1.780968, forward_loss: 0.000000, inverse_pred_loss: 1.780527, Q_loss: 0.000441, intrinsic_rewards: 0.000001, epsilon: 0.010055, episode:  225\n",
      "frames: 491000, reward: 17.300000, total_loss: 1.785949, forward_loss: 0.000000, inverse_pred_loss: 1.785719, Q_loss: 0.000230, intrinsic_rewards: 0.000001, epsilon: 0.010054, episode:  226\n",
      "frames: 492000, reward: 17.300000, total_loss: 1.814086, forward_loss: 0.000000, inverse_pred_loss: 1.813405, Q_loss: 0.000681, intrinsic_rewards: 0.000001, epsilon: 0.010053, episode:  226\n",
      "frames: 493000, reward: 17.700000, total_loss: 1.828247, forward_loss: 0.000000, inverse_pred_loss: 1.827980, Q_loss: 0.000268, intrinsic_rewards: 0.000000, epsilon: 0.010052, episode:  227\n",
      "frames: 494000, reward: 17.700000, total_loss: 1.830256, forward_loss: 0.000000, inverse_pred_loss: 1.830112, Q_loss: 0.000144, intrinsic_rewards: 0.000000, epsilon: 0.010051, episode:  227\n",
      "frames: 495000, reward: 17.800000, total_loss: 1.784498, forward_loss: 0.000000, inverse_pred_loss: 1.784403, Q_loss: 0.000095, intrinsic_rewards: 0.000001, epsilon: 0.010050, episode:  228\n",
      "frames: 496000, reward: 17.800000, total_loss: 1.777666, forward_loss: 0.000000, inverse_pred_loss: 1.777441, Q_loss: 0.000225, intrinsic_rewards: 0.000000, epsilon: 0.010049, episode:  228\n",
      "frames: 497000, reward: 17.900000, total_loss: 1.785036, forward_loss: 0.000000, inverse_pred_loss: 1.784782, Q_loss: 0.000255, intrinsic_rewards: 0.000002, epsilon: 0.010048, episode:  229\n",
      "frames: 498000, reward: 17.900000, total_loss: 1.808829, forward_loss: 0.000000, inverse_pred_loss: 1.808611, Q_loss: 0.000218, intrinsic_rewards: 0.000002, epsilon: 0.010047, episode:  229\n",
      "frames: 499000, reward: 17.500000, total_loss: 1.803667, forward_loss: 0.000000, inverse_pred_loss: 1.803239, Q_loss: 0.000427, intrinsic_rewards: 0.000001, epsilon: 0.010046, episode:  230\n",
      "frames: 500000, reward: 17.500000, total_loss: 1.816157, forward_loss: 0.000000, inverse_pred_loss: 1.815865, Q_loss: 0.000291, intrinsic_rewards: 0.000002, epsilon: 0.010045, episode:  230\n",
      "frames: 501000, reward: 16.700000, total_loss: 1.819337, forward_loss: 0.000000, inverse_pred_loss: 1.818916, Q_loss: 0.000421, intrinsic_rewards: 0.000000, epsilon: 0.010044, episode:  231\n",
      "frames: 502000, reward: 16.700000, total_loss: 1.781907, forward_loss: 0.000000, inverse_pred_loss: 1.781460, Q_loss: 0.000447, intrinsic_rewards: 0.000003, epsilon: 0.010043, episode:  231\n",
      "frames: 503000, reward: 16.700000, total_loss: 1.768697, forward_loss: 0.000000, inverse_pred_loss: 1.767336, Q_loss: 0.001361, intrinsic_rewards: 0.000002, epsilon: 0.010042, episode:  231\n",
      "frames: 504000, reward: 15.800000, total_loss: 1.777715, forward_loss: 0.000000, inverse_pred_loss: 1.777225, Q_loss: 0.000490, intrinsic_rewards: 0.000002, epsilon: 0.010041, episode:  232\n",
      "frames: 505000, reward: 15.800000, total_loss: 1.771945, forward_loss: 0.000000, inverse_pred_loss: 1.771633, Q_loss: 0.000311, intrinsic_rewards: 0.000003, epsilon: 0.010041, episode:  232\n",
      "frames: 506000, reward: 15.700000, total_loss: 1.799313, forward_loss: 0.000000, inverse_pred_loss: 1.799141, Q_loss: 0.000172, intrinsic_rewards: 0.000000, epsilon: 0.010040, episode:  233\n",
      "frames: 507000, reward: 15.700000, total_loss: 1.800771, forward_loss: 0.000000, inverse_pred_loss: 1.799917, Q_loss: 0.000855, intrinsic_rewards: 0.000001, epsilon: 0.010039, episode:  233\n",
      "frames: 508000, reward: 15.700000, total_loss: 1.786153, forward_loss: 0.000000, inverse_pred_loss: 1.785653, Q_loss: 0.000500, intrinsic_rewards: 0.000001, epsilon: 0.010038, episode:  234\n",
      "frames: 509000, reward: 15.700000, total_loss: 1.784443, forward_loss: 0.000000, inverse_pred_loss: 1.784039, Q_loss: 0.000404, intrinsic_rewards: 0.000000, epsilon: 0.010038, episode:  234\n",
      "frames: 510000, reward: 15.900000, total_loss: 1.792104, forward_loss: 0.000000, inverse_pred_loss: 1.791474, Q_loss: 0.000630, intrinsic_rewards: 0.000000, epsilon: 0.010037, episode:  235\n",
      "frames: 511000, reward: 15.900000, total_loss: 1.808633, forward_loss: 0.000000, inverse_pred_loss: 1.808424, Q_loss: 0.000210, intrinsic_rewards: 0.000000, epsilon: 0.010036, episode:  235\n",
      "frames: 512000, reward: 15.700000, total_loss: 1.803890, forward_loss: 0.000000, inverse_pred_loss: 1.803578, Q_loss: 0.000312, intrinsic_rewards: 0.000000, epsilon: 0.010035, episode:  236\n",
      "frames: 513000, reward: 15.900000, total_loss: 1.801811, forward_loss: 0.000000, inverse_pred_loss: 1.801534, Q_loss: 0.000278, intrinsic_rewards: 0.000000, epsilon: 0.010035, episode:  237\n",
      "frames: 514000, reward: 15.900000, total_loss: 1.813513, forward_loss: 0.000000, inverse_pred_loss: 1.813389, Q_loss: 0.000124, intrinsic_rewards: 0.000000, epsilon: 0.010034, episode:  237\n",
      "frames: 515000, reward: 15.900000, total_loss: 1.801549, forward_loss: 0.000000, inverse_pred_loss: 1.801340, Q_loss: 0.000209, intrinsic_rewards: 0.000000, epsilon: 0.010033, episode:  237\n",
      "frames: 516000, reward: 15.600000, total_loss: 1.827382, forward_loss: 0.000000, inverse_pred_loss: 1.827032, Q_loss: 0.000349, intrinsic_rewards: 0.000000, epsilon: 0.010033, episode:  238\n",
      "frames: 517000, reward: 15.700000, total_loss: 1.748631, forward_loss: 0.000000, inverse_pred_loss: 1.748451, Q_loss: 0.000180, intrinsic_rewards: 0.000000, epsilon: 0.010032, episode:  239\n",
      "frames: 518000, reward: 15.700000, total_loss: 1.799778, forward_loss: 0.000000, inverse_pred_loss: 1.799608, Q_loss: 0.000169, intrinsic_rewards: 0.000000, epsilon: 0.010031, episode:  239\n",
      "frames: 519000, reward: 15.700000, total_loss: 1.815696, forward_loss: 0.000000, inverse_pred_loss: 1.815567, Q_loss: 0.000129, intrinsic_rewards: 0.000000, epsilon: 0.010031, episode:  239\n",
      "frames: 520000, reward: 15.800000, total_loss: 1.756444, forward_loss: 0.000000, inverse_pred_loss: 1.756266, Q_loss: 0.000177, intrinsic_rewards: 0.000000, epsilon: 0.010030, episode:  240\n",
      "frames: 521000, reward: 16.600000, total_loss: 1.798214, forward_loss: 0.000000, inverse_pred_loss: 1.798098, Q_loss: 0.000116, intrinsic_rewards: 0.000000, epsilon: 0.010030, episode:  241\n",
      "frames: 522000, reward: 16.600000, total_loss: 1.778569, forward_loss: 0.000000, inverse_pred_loss: 1.778383, Q_loss: 0.000185, intrinsic_rewards: 0.000000, epsilon: 0.010029, episode:  241\n",
      "frames: 523000, reward: 17.900000, total_loss: 1.814223, forward_loss: 0.000000, inverse_pred_loss: 1.813875, Q_loss: 0.000348, intrinsic_rewards: 0.000000, epsilon: 0.010028, episode:  242\n",
      "frames: 524000, reward: 17.900000, total_loss: 1.782670, forward_loss: 0.000000, inverse_pred_loss: 1.782541, Q_loss: 0.000129, intrinsic_rewards: 0.000000, epsilon: 0.010028, episode:  242\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 525000, reward: 17.900000, total_loss: 1.791449, forward_loss: 0.000000, inverse_pred_loss: 1.791301, Q_loss: 0.000148, intrinsic_rewards: 0.000000, epsilon: 0.010027, episode:  242\n",
      "frames: 526000, reward: 17.200000, total_loss: 1.779116, forward_loss: 0.000000, inverse_pred_loss: 1.778844, Q_loss: 0.000272, intrinsic_rewards: 0.000000, epsilon: 0.010027, episode:  243\n",
      "frames: 527000, reward: 17.200000, total_loss: 1.792431, forward_loss: 0.000000, inverse_pred_loss: 1.792243, Q_loss: 0.000188, intrinsic_rewards: 0.000000, epsilon: 0.010026, episode:  244\n",
      "frames: 528000, reward: 17.200000, total_loss: 1.789570, forward_loss: 0.000000, inverse_pred_loss: 1.789410, Q_loss: 0.000160, intrinsic_rewards: 0.000000, epsilon: 0.010026, episode:  244\n",
      "frames: 529000, reward: 17.400000, total_loss: 1.788007, forward_loss: 0.000000, inverse_pred_loss: 1.787845, Q_loss: 0.000162, intrinsic_rewards: 0.000000, epsilon: 0.010025, episode:  245\n",
      "frames: 530000, reward: 17.400000, total_loss: 1.774141, forward_loss: 0.000000, inverse_pred_loss: 1.774015, Q_loss: 0.000126, intrinsic_rewards: 0.000000, epsilon: 0.010025, episode:  245\n",
      "frames: 531000, reward: 17.500000, total_loss: 1.804163, forward_loss: 0.000000, inverse_pred_loss: 1.803926, Q_loss: 0.000237, intrinsic_rewards: 0.000000, epsilon: 0.010024, episode:  246\n",
      "frames: 532000, reward: 17.500000, total_loss: 1.791593, forward_loss: 0.000000, inverse_pred_loss: 1.791504, Q_loss: 0.000089, intrinsic_rewards: 0.000000, epsilon: 0.010024, episode:  246\n",
      "frames: 533000, reward: 17.300000, total_loss: 1.745422, forward_loss: 0.000000, inverse_pred_loss: 1.745250, Q_loss: 0.000172, intrinsic_rewards: 0.000000, epsilon: 0.010023, episode:  247\n",
      "frames: 534000, reward: 17.300000, total_loss: 1.818746, forward_loss: 0.000000, inverse_pred_loss: 1.818215, Q_loss: 0.000531, intrinsic_rewards: 0.000000, epsilon: 0.010023, episode:  247\n",
      "frames: 535000, reward: 17.300000, total_loss: 1.785892, forward_loss: 0.000000, inverse_pred_loss: 1.785441, Q_loss: 0.000451, intrinsic_rewards: 0.000000, epsilon: 0.010022, episode:  247\n",
      "frames: 536000, reward: 16.600000, total_loss: 1.755087, forward_loss: 0.000000, inverse_pred_loss: 1.754641, Q_loss: 0.000446, intrinsic_rewards: 0.000000, epsilon: 0.010022, episode:  248\n",
      "frames: 537000, reward: 16.600000, total_loss: 1.805601, forward_loss: 0.000000, inverse_pred_loss: 1.805320, Q_loss: 0.000282, intrinsic_rewards: 0.000000, epsilon: 0.010021, episode:  248\n",
      "frames: 538000, reward: 16.300000, total_loss: 1.755866, forward_loss: 0.000000, inverse_pred_loss: 1.755723, Q_loss: 0.000143, intrinsic_rewards: 0.000000, epsilon: 0.010021, episode:  249\n",
      "frames: 539000, reward: 16.800000, total_loss: 1.784688, forward_loss: 0.000000, inverse_pred_loss: 1.784168, Q_loss: 0.000520, intrinsic_rewards: 0.000000, epsilon: 0.010021, episode:  250\n",
      "frames: 540000, reward: 16.800000, total_loss: 1.769663, forward_loss: 0.000000, inverse_pred_loss: 1.769422, Q_loss: 0.000242, intrinsic_rewards: 0.000000, epsilon: 0.010020, episode:  250\n",
      "frames: 541000, reward: 16.500000, total_loss: 1.720460, forward_loss: 0.000000, inverse_pred_loss: 1.720226, Q_loss: 0.000233, intrinsic_rewards: 0.000000, epsilon: 0.010020, episode:  251\n",
      "frames: 542000, reward: 16.500000, total_loss: 1.748863, forward_loss: 0.000000, inverse_pred_loss: 1.748639, Q_loss: 0.000224, intrinsic_rewards: 0.000000, epsilon: 0.010019, episode:  251\n",
      "frames: 543000, reward: 16.200000, total_loss: 1.800217, forward_loss: 0.000000, inverse_pred_loss: 1.800057, Q_loss: 0.000160, intrinsic_rewards: 0.000000, epsilon: 0.010019, episode:  252\n",
      "frames: 544000, reward: 16.200000, total_loss: 1.767293, forward_loss: 0.000000, inverse_pred_loss: 1.767115, Q_loss: 0.000179, intrinsic_rewards: 0.000000, epsilon: 0.010019, episode:  252\n",
      "frames: 545000, reward: 16.700000, total_loss: 1.776527, forward_loss: 0.000000, inverse_pred_loss: 1.776322, Q_loss: 0.000204, intrinsic_rewards: 0.000000, epsilon: 0.010018, episode:  253\n",
      "frames: 546000, reward: 16.700000, total_loss: 1.743936, forward_loss: 0.000000, inverse_pred_loss: 1.743734, Q_loss: 0.000202, intrinsic_rewards: 0.000000, epsilon: 0.010018, episode:  253\n",
      "frames: 547000, reward: 16.700000, total_loss: 1.761438, forward_loss: 0.000000, inverse_pred_loss: 1.761292, Q_loss: 0.000147, intrinsic_rewards: 0.000000, epsilon: 0.010018, episode:  253\n",
      "frames: 548000, reward: 15.600000, total_loss: 1.751627, forward_loss: 0.000000, inverse_pred_loss: 1.751028, Q_loss: 0.000600, intrinsic_rewards: 0.000000, epsilon: 0.010017, episode:  254\n",
      "frames: 549000, reward: 15.600000, total_loss: 1.771597, forward_loss: 0.000000, inverse_pred_loss: 1.771434, Q_loss: 0.000164, intrinsic_rewards: 0.000000, epsilon: 0.010017, episode:  254\n",
      "frames: 550000, reward: 15.600000, total_loss: 1.790160, forward_loss: 0.000000, inverse_pred_loss: 1.790066, Q_loss: 0.000095, intrinsic_rewards: 0.000000, epsilon: 0.010017, episode:  255\n",
      "frames: 551000, reward: 15.600000, total_loss: 1.823767, forward_loss: 0.000000, inverse_pred_loss: 1.823613, Q_loss: 0.000154, intrinsic_rewards: 0.000000, epsilon: 0.010016, episode:  255\n",
      "frames: 552000, reward: 15.600000, total_loss: 1.790758, forward_loss: 0.000000, inverse_pred_loss: 1.790247, Q_loss: 0.000510, intrinsic_rewards: 0.000000, epsilon: 0.010016, episode:  256\n",
      "frames: 553000, reward: 15.600000, total_loss: 1.784455, forward_loss: 0.000000, inverse_pred_loss: 1.783772, Q_loss: 0.000684, intrinsic_rewards: 0.000000, epsilon: 0.010016, episode:  256\n",
      "frames: 554000, reward: 14.700000, total_loss: 1.747722, forward_loss: 0.000000, inverse_pred_loss: 1.747525, Q_loss: 0.000197, intrinsic_rewards: 0.000000, epsilon: 0.010015, episode:  257\n",
      "frames: 555000, reward: 14.700000, total_loss: 1.813882, forward_loss: 0.000000, inverse_pred_loss: 1.813742, Q_loss: 0.000140, intrinsic_rewards: 0.000000, epsilon: 0.010015, episode:  257\n",
      "frames: 556000, reward: 15.700000, total_loss: 1.806246, forward_loss: 0.000000, inverse_pred_loss: 1.806004, Q_loss: 0.000242, intrinsic_rewards: 0.000000, epsilon: 0.010015, episode:  258\n",
      "frames: 557000, reward: 15.700000, total_loss: 1.823135, forward_loss: 0.000000, inverse_pred_loss: 1.822779, Q_loss: 0.000357, intrinsic_rewards: 0.000000, epsilon: 0.010014, episode:  258\n",
      "frames: 558000, reward: 16.000000, total_loss: 1.764982, forward_loss: 0.000000, inverse_pred_loss: 1.764617, Q_loss: 0.000365, intrinsic_rewards: 0.000000, epsilon: 0.010014, episode:  259\n",
      "frames: 559000, reward: 16.000000, total_loss: 1.768354, forward_loss: 0.000000, inverse_pred_loss: 1.768165, Q_loss: 0.000189, intrinsic_rewards: 0.000000, epsilon: 0.010014, episode:  259\n",
      "frames: 560000, reward: 15.900000, total_loss: 1.797777, forward_loss: 0.000000, inverse_pred_loss: 1.797494, Q_loss: 0.000283, intrinsic_rewards: 0.000000, epsilon: 0.010014, episode:  260\n",
      "frames: 561000, reward: 15.900000, total_loss: 1.762959, forward_loss: 0.000000, inverse_pred_loss: 1.761812, Q_loss: 0.001147, intrinsic_rewards: 0.000000, epsilon: 0.010013, episode:  260\n",
      "frames: 562000, reward: 15.100000, total_loss: 1.764827, forward_loss: 0.000000, inverse_pred_loss: 1.764685, Q_loss: 0.000142, intrinsic_rewards: 0.000000, epsilon: 0.010013, episode:  261\n",
      "frames: 563000, reward: 15.100000, total_loss: 1.782693, forward_loss: 0.000000, inverse_pred_loss: 1.782554, Q_loss: 0.000139, intrinsic_rewards: 0.000000, epsilon: 0.010013, episode:  261\n",
      "frames: 564000, reward: 15.300000, total_loss: 1.764943, forward_loss: 0.000000, inverse_pred_loss: 1.764618, Q_loss: 0.000324, intrinsic_rewards: 0.000000, epsilon: 0.010012, episode:  262\n",
      "frames: 565000, reward: 15.300000, total_loss: 1.804878, forward_loss: 0.000000, inverse_pred_loss: 1.804562, Q_loss: 0.000316, intrinsic_rewards: 0.000000, epsilon: 0.010012, episode:  262\n",
      "frames: 566000, reward: 15.600000, total_loss: 1.779284, forward_loss: 0.000000, inverse_pred_loss: 1.779099, Q_loss: 0.000185, intrinsic_rewards: 0.000000, epsilon: 0.010012, episode:  263\n",
      "frames: 567000, reward: 15.600000, total_loss: 1.794535, forward_loss: 0.000000, inverse_pred_loss: 1.794336, Q_loss: 0.000198, intrinsic_rewards: 0.000000, epsilon: 0.010012, episode:  263\n",
      "frames: 568000, reward: 16.400000, total_loss: 1.766060, forward_loss: 0.000000, inverse_pred_loss: 1.765825, Q_loss: 0.000235, intrinsic_rewards: 0.000000, epsilon: 0.010012, episode:  264\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 569000, reward: 16.400000, total_loss: 1.754086, forward_loss: 0.000000, inverse_pred_loss: 1.753810, Q_loss: 0.000275, intrinsic_rewards: 0.000000, epsilon: 0.010011, episode:  264\n",
      "frames: 570000, reward: 16.000000, total_loss: 1.744100, forward_loss: 0.000000, inverse_pred_loss: 1.743999, Q_loss: 0.000100, intrinsic_rewards: 0.000000, epsilon: 0.010011, episode:  265\n",
      "frames: 571000, reward: 16.000000, total_loss: 1.792853, forward_loss: 0.000000, inverse_pred_loss: 1.792666, Q_loss: 0.000186, intrinsic_rewards: 0.000000, epsilon: 0.010011, episode:  265\n",
      "frames: 572000, reward: 16.100000, total_loss: 1.771085, forward_loss: 0.000000, inverse_pred_loss: 1.770827, Q_loss: 0.000257, intrinsic_rewards: 0.000000, epsilon: 0.010011, episode:  266\n",
      "frames: 573000, reward: 16.100000, total_loss: 1.776825, forward_loss: 0.000000, inverse_pred_loss: 1.776706, Q_loss: 0.000119, intrinsic_rewards: 0.000000, epsilon: 0.010010, episode:  266\n",
      "frames: 574000, reward: 16.000000, total_loss: 1.783048, forward_loss: 0.000000, inverse_pred_loss: 1.782765, Q_loss: 0.000284, intrinsic_rewards: 0.000001, epsilon: 0.010010, episode:  267\n",
      "frames: 575000, reward: 16.000000, total_loss: 1.799939, forward_loss: 0.000000, inverse_pred_loss: 1.799823, Q_loss: 0.000116, intrinsic_rewards: 0.000000, epsilon: 0.010010, episode:  267\n",
      "frames: 576000, reward: 16.300000, total_loss: 1.796225, forward_loss: 0.000000, inverse_pred_loss: 1.796087, Q_loss: 0.000138, intrinsic_rewards: 0.000000, epsilon: 0.010010, episode:  268\n",
      "frames: 577000, reward: 16.300000, total_loss: 1.785192, forward_loss: 0.000000, inverse_pred_loss: 1.785018, Q_loss: 0.000173, intrinsic_rewards: 0.000001, epsilon: 0.010010, episode:  268\n",
      "frames: 578000, reward: 16.400000, total_loss: 1.770179, forward_loss: 0.000000, inverse_pred_loss: 1.764405, Q_loss: 0.005774, intrinsic_rewards: 0.000000, epsilon: 0.010009, episode:  269\n",
      "frames: 579000, reward: 16.400000, total_loss: 1.747513, forward_loss: 0.000000, inverse_pred_loss: 1.747409, Q_loss: 0.000104, intrinsic_rewards: 0.000000, epsilon: 0.010009, episode:  269\n",
      "frames: 580000, reward: 16.000000, total_loss: 1.760098, forward_loss: 0.000000, inverse_pred_loss: 1.759925, Q_loss: 0.000173, intrinsic_rewards: 0.000000, epsilon: 0.010009, episode:  270\n",
      "frames: 581000, reward: 16.000000, total_loss: 1.773330, forward_loss: 0.000000, inverse_pred_loss: 1.773033, Q_loss: 0.000297, intrinsic_rewards: 0.000000, epsilon: 0.010009, episode:  270\n",
      "frames: 582000, reward: 16.700000, total_loss: 1.800827, forward_loss: 0.000000, inverse_pred_loss: 1.800664, Q_loss: 0.000163, intrinsic_rewards: 0.000000, epsilon: 0.010009, episode:  271\n",
      "frames: 583000, reward: 17.100000, total_loss: 1.798322, forward_loss: 0.000000, inverse_pred_loss: 1.798139, Q_loss: 0.000183, intrinsic_rewards: 0.000000, epsilon: 0.010009, episode:  272\n",
      "frames: 584000, reward: 17.100000, total_loss: 1.729590, forward_loss: 0.000000, inverse_pred_loss: 1.729353, Q_loss: 0.000237, intrinsic_rewards: 0.000000, epsilon: 0.010008, episode:  272\n",
      "frames: 585000, reward: 17.100000, total_loss: 1.793998, forward_loss: 0.000000, inverse_pred_loss: 1.793704, Q_loss: 0.000294, intrinsic_rewards: 0.000000, epsilon: 0.010008, episode:  273\n",
      "frames: 586000, reward: 17.100000, total_loss: 1.806070, forward_loss: 0.000000, inverse_pred_loss: 1.805851, Q_loss: 0.000219, intrinsic_rewards: 0.000000, epsilon: 0.010008, episode:  273\n",
      "frames: 587000, reward: 17.500000, total_loss: 1.775077, forward_loss: 0.000000, inverse_pred_loss: 1.774956, Q_loss: 0.000121, intrinsic_rewards: 0.000000, epsilon: 0.010008, episode:  274\n",
      "frames: 588000, reward: 17.500000, total_loss: 1.780340, forward_loss: 0.000000, inverse_pred_loss: 1.779974, Q_loss: 0.000366, intrinsic_rewards: 0.000000, epsilon: 0.010008, episode:  274\n",
      "frames: 589000, reward: 17.800000, total_loss: 1.771343, forward_loss: 0.000000, inverse_pred_loss: 1.771238, Q_loss: 0.000106, intrinsic_rewards: 0.000000, epsilon: 0.010008, episode:  275\n",
      "frames: 590000, reward: 17.800000, total_loss: 1.753477, forward_loss: 0.000000, inverse_pred_loss: 1.753378, Q_loss: 0.000099, intrinsic_rewards: 0.000000, epsilon: 0.010007, episode:  275\n",
      "frames: 591000, reward: 17.900000, total_loss: 1.728354, forward_loss: 0.000000, inverse_pred_loss: 1.728183, Q_loss: 0.000171, intrinsic_rewards: 0.000000, epsilon: 0.010007, episode:  276\n",
      "frames: 592000, reward: 17.900000, total_loss: 1.800529, forward_loss: 0.000000, inverse_pred_loss: 1.800416, Q_loss: 0.000113, intrinsic_rewards: 0.000000, epsilon: 0.010007, episode:  276\n",
      "frames: 593000, reward: 18.900000, total_loss: 1.763140, forward_loss: 0.000000, inverse_pred_loss: 1.762963, Q_loss: 0.000176, intrinsic_rewards: 0.000000, epsilon: 0.010007, episode:  277\n",
      "frames: 594000, reward: 18.900000, total_loss: 1.738916, forward_loss: 0.000000, inverse_pred_loss: 1.738741, Q_loss: 0.000175, intrinsic_rewards: 0.000000, epsilon: 0.010007, episode:  278\n",
      "frames: 595000, reward: 18.900000, total_loss: 1.716807, forward_loss: 0.000000, inverse_pred_loss: 1.716707, Q_loss: 0.000100, intrinsic_rewards: 0.000000, epsilon: 0.010007, episode:  278\n",
      "frames: 596000, reward: 19.000000, total_loss: 1.775971, forward_loss: 0.000000, inverse_pred_loss: 1.775868, Q_loss: 0.000104, intrinsic_rewards: 0.000000, epsilon: 0.010007, episode:  279\n",
      "frames: 597000, reward: 19.000000, total_loss: 1.790408, forward_loss: 0.000000, inverse_pred_loss: 1.790330, Q_loss: 0.000078, intrinsic_rewards: 0.000000, epsilon: 0.010006, episode:  279\n",
      "frames: 598000, reward: 19.700000, total_loss: 1.745224, forward_loss: 0.000000, inverse_pred_loss: 1.745062, Q_loss: 0.000162, intrinsic_rewards: 0.000000, epsilon: 0.010006, episode:  280\n",
      "frames: 599000, reward: 19.700000, total_loss: 1.773835, forward_loss: 0.000000, inverse_pred_loss: 1.773770, Q_loss: 0.000065, intrinsic_rewards: 0.000000, epsilon: 0.010006, episode:  280\n",
      "frames: 600000, reward: 19.900000, total_loss: 1.746118, forward_loss: 0.000000, inverse_pred_loss: 1.745961, Q_loss: 0.000157, intrinsic_rewards: 0.000000, epsilon: 0.010006, episode:  281\n",
      "frames: 601000, reward: 19.900000, total_loss: 1.833829, forward_loss: 0.000000, inverse_pred_loss: 1.833738, Q_loss: 0.000091, intrinsic_rewards: 0.000000, epsilon: 0.010006, episode:  281\n",
      "frames: 602000, reward: 19.500000, total_loss: 1.811337, forward_loss: 0.000000, inverse_pred_loss: 1.811279, Q_loss: 0.000058, intrinsic_rewards: 0.000000, epsilon: 0.010006, episode:  282\n",
      "frames: 603000, reward: 19.500000, total_loss: 1.818905, forward_loss: 0.000000, inverse_pred_loss: 1.818676, Q_loss: 0.000229, intrinsic_rewards: 0.000000, epsilon: 0.010006, episode:  283\n",
      "frames: 604000, reward: 19.500000, total_loss: 1.753691, forward_loss: 0.000000, inverse_pred_loss: 1.753636, Q_loss: 0.000055, intrinsic_rewards: 0.000000, epsilon: 0.010006, episode:  283\n",
      "frames: 605000, reward: 19.500000, total_loss: 1.783898, forward_loss: 0.000000, inverse_pred_loss: 1.783838, Q_loss: 0.000060, intrinsic_rewards: 0.000000, epsilon: 0.010006, episode:  284\n",
      "frames: 606000, reward: 19.500000, total_loss: 1.732692, forward_loss: 0.000000, inverse_pred_loss: 1.732599, Q_loss: 0.000094, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  284\n",
      "frames: 607000, reward: 19.500000, total_loss: 1.715556, forward_loss: 0.000000, inverse_pred_loss: 1.715490, Q_loss: 0.000066, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  285\n",
      "frames: 608000, reward: 19.500000, total_loss: 1.715163, forward_loss: 0.000000, inverse_pred_loss: 1.715120, Q_loss: 0.000043, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  285\n",
      "frames: 609000, reward: 19.500000, total_loss: 1.766108, forward_loss: 0.000000, inverse_pred_loss: 1.766042, Q_loss: 0.000066, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  286\n",
      "frames: 610000, reward: 19.400000, total_loss: 1.738377, forward_loss: 0.000000, inverse_pred_loss: 1.738279, Q_loss: 0.000098, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  287\n",
      "frames: 611000, reward: 19.400000, total_loss: 1.774276, forward_loss: 0.000000, inverse_pred_loss: 1.774179, Q_loss: 0.000097, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  287\n",
      "frames: 612000, reward: 19.400000, total_loss: 1.785208, forward_loss: 0.000000, inverse_pred_loss: 1.784937, Q_loss: 0.000271, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  287\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 613000, reward: 18.700000, total_loss: 1.744048, forward_loss: 0.000000, inverse_pred_loss: 1.743914, Q_loss: 0.000133, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  288\n",
      "frames: 614000, reward: 18.700000, total_loss: 1.747671, forward_loss: 0.000000, inverse_pred_loss: 1.747358, Q_loss: 0.000312, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  288\n",
      "frames: 615000, reward: 17.700000, total_loss: 1.770988, forward_loss: 0.000000, inverse_pred_loss: 1.770046, Q_loss: 0.000942, intrinsic_rewards: 0.000000, epsilon: 0.010005, episode:  289\n",
      "frames: 616000, reward: 17.700000, total_loss: 1.778439, forward_loss: 0.000000, inverse_pred_loss: 1.778303, Q_loss: 0.000136, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  289\n",
      "frames: 617000, reward: 17.200000, total_loss: 1.769890, forward_loss: 0.000000, inverse_pred_loss: 1.769537, Q_loss: 0.000352, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  290\n",
      "frames: 618000, reward: 17.200000, total_loss: 1.824307, forward_loss: 0.000000, inverse_pred_loss: 1.824059, Q_loss: 0.000248, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  290\n",
      "frames: 619000, reward: 17.400000, total_loss: 1.762614, forward_loss: 0.000000, inverse_pred_loss: 1.762338, Q_loss: 0.000275, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  291\n",
      "frames: 620000, reward: 17.400000, total_loss: 1.767849, forward_loss: 0.000000, inverse_pred_loss: 1.767496, Q_loss: 0.000353, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  291\n",
      "frames: 621000, reward: 16.800000, total_loss: 1.716846, forward_loss: 0.000000, inverse_pred_loss: 1.716642, Q_loss: 0.000204, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  292\n",
      "frames: 622000, reward: 16.800000, total_loss: 1.811310, forward_loss: 0.000000, inverse_pred_loss: 1.811006, Q_loss: 0.000304, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  292\n",
      "frames: 623000, reward: 16.800000, total_loss: 1.795944, forward_loss: 0.000000, inverse_pred_loss: 1.795821, Q_loss: 0.000123, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  293\n",
      "frames: 624000, reward: 16.800000, total_loss: 1.806250, forward_loss: 0.000000, inverse_pred_loss: 1.806100, Q_loss: 0.000150, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  293\n",
      "frames: 625000, reward: 16.800000, total_loss: 1.764699, forward_loss: 0.000000, inverse_pred_loss: 1.764587, Q_loss: 0.000112, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  294\n",
      "frames: 626000, reward: 16.900000, total_loss: 1.766806, forward_loss: 0.000000, inverse_pred_loss: 1.766681, Q_loss: 0.000124, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  295\n",
      "frames: 627000, reward: 16.900000, total_loss: 1.795630, forward_loss: 0.000000, inverse_pred_loss: 1.795533, Q_loss: 0.000098, intrinsic_rewards: 0.000000, epsilon: 0.010004, episode:  295\n",
      "frames: 628000, reward: 16.400000, total_loss: 1.779481, forward_loss: 0.000000, inverse_pred_loss: 1.778888, Q_loss: 0.000592, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  296\n",
      "frames: 629000, reward: 16.400000, total_loss: 1.764661, forward_loss: 0.000000, inverse_pred_loss: 1.764400, Q_loss: 0.000261, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  296\n",
      "frames: 630000, reward: 16.700000, total_loss: 1.796601, forward_loss: 0.000000, inverse_pred_loss: 1.795891, Q_loss: 0.000710, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  297\n",
      "frames: 631000, reward: 16.700000, total_loss: 1.765462, forward_loss: 0.000000, inverse_pred_loss: 1.765358, Q_loss: 0.000104, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  297\n",
      "frames: 632000, reward: 16.600000, total_loss: 1.803115, forward_loss: 0.000000, inverse_pred_loss: 1.803011, Q_loss: 0.000103, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  298\n",
      "frames: 633000, reward: 16.600000, total_loss: 1.784566, forward_loss: 0.000000, inverse_pred_loss: 1.784252, Q_loss: 0.000314, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  298\n",
      "frames: 634000, reward: 17.300000, total_loss: 1.795599, forward_loss: 0.000000, inverse_pred_loss: 1.795511, Q_loss: 0.000088, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  299\n",
      "frames: 635000, reward: 17.300000, total_loss: 1.756583, forward_loss: 0.000000, inverse_pred_loss: 1.756435, Q_loss: 0.000148, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  299\n",
      "frames: 636000, reward: 17.600000, total_loss: 1.790815, forward_loss: 0.000000, inverse_pred_loss: 1.790294, Q_loss: 0.000521, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  300\n",
      "frames: 637000, reward: 17.600000, total_loss: 1.751174, forward_loss: 0.000000, inverse_pred_loss: 1.751110, Q_loss: 0.000064, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  300\n",
      "frames: 638000, reward: 17.600000, total_loss: 1.806924, forward_loss: 0.000000, inverse_pred_loss: 1.805794, Q_loss: 0.001131, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  301\n",
      "frames: 639000, reward: 17.600000, total_loss: 1.820206, forward_loss: 0.000000, inverse_pred_loss: 1.820093, Q_loss: 0.000113, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  301\n",
      "frames: 640000, reward: 17.300000, total_loss: 1.782379, forward_loss: 0.000000, inverse_pred_loss: 1.782249, Q_loss: 0.000130, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  302\n",
      "frames: 641000, reward: 17.300000, total_loss: 1.731252, forward_loss: 0.000000, inverse_pred_loss: 1.731043, Q_loss: 0.000209, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  302\n",
      "frames: 642000, reward: 17.300000, total_loss: 1.778452, forward_loss: 0.000000, inverse_pred_loss: 1.778238, Q_loss: 0.000214, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  303\n",
      "frames: 643000, reward: 17.300000, total_loss: 1.799158, forward_loss: 0.000000, inverse_pred_loss: 1.798860, Q_loss: 0.000298, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  303\n",
      "frames: 644000, reward: 17.300000, total_loss: 1.748156, forward_loss: 0.000000, inverse_pred_loss: 1.747850, Q_loss: 0.000307, intrinsic_rewards: 0.000000, epsilon: 0.010003, episode:  303\n",
      "frames: 645000, reward: 16.400000, total_loss: 1.743900, forward_loss: 0.000000, inverse_pred_loss: 1.743786, Q_loss: 0.000114, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  304\n",
      "frames: 646000, reward: 16.400000, total_loss: 1.780652, forward_loss: 0.000000, inverse_pred_loss: 1.780381, Q_loss: 0.000271, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  305\n",
      "frames: 647000, reward: 16.400000, total_loss: 1.788900, forward_loss: 0.000000, inverse_pred_loss: 1.788782, Q_loss: 0.000119, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  305\n",
      "frames: 648000, reward: 16.900000, total_loss: 1.773379, forward_loss: 0.000000, inverse_pred_loss: 1.773223, Q_loss: 0.000156, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  306\n",
      "frames: 649000, reward: 16.900000, total_loss: 1.764924, forward_loss: 0.000000, inverse_pred_loss: 1.764796, Q_loss: 0.000128, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  306\n",
      "frames: 650000, reward: 16.900000, total_loss: 1.746485, forward_loss: 0.000000, inverse_pred_loss: 1.746285, Q_loss: 0.000200, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  307\n",
      "frames: 651000, reward: 16.900000, total_loss: 1.750028, forward_loss: 0.000000, inverse_pred_loss: 1.749833, Q_loss: 0.000195, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  307\n",
      "frames: 652000, reward: 17.600000, total_loss: 1.740690, forward_loss: 0.000000, inverse_pred_loss: 1.740479, Q_loss: 0.000211, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  308\n",
      "frames: 653000, reward: 17.600000, total_loss: 1.793425, forward_loss: 0.000000, inverse_pred_loss: 1.793205, Q_loss: 0.000220, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  308\n",
      "frames: 654000, reward: 17.300000, total_loss: 1.764982, forward_loss: 0.000000, inverse_pred_loss: 1.764860, Q_loss: 0.000122, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  309\n",
      "frames: 655000, reward: 17.300000, total_loss: 1.774900, forward_loss: 0.000000, inverse_pred_loss: 1.774778, Q_loss: 0.000121, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  309\n",
      "frames: 656000, reward: 17.400000, total_loss: 1.750052, forward_loss: 0.000000, inverse_pred_loss: 1.749944, Q_loss: 0.000108, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  310\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 657000, reward: 17.400000, total_loss: 1.776552, forward_loss: 0.000000, inverse_pred_loss: 1.776353, Q_loss: 0.000199, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  310\n",
      "frames: 658000, reward: 17.100000, total_loss: 1.790858, forward_loss: 0.000000, inverse_pred_loss: 1.790702, Q_loss: 0.000156, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  311\n",
      "frames: 659000, reward: 18.200000, total_loss: 1.746806, forward_loss: 0.000000, inverse_pred_loss: 1.746667, Q_loss: 0.000140, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  312\n",
      "frames: 660000, reward: 18.200000, total_loss: 1.782716, forward_loss: 0.000000, inverse_pred_loss: 1.782554, Q_loss: 0.000162, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  312\n",
      "frames: 661000, reward: 18.200000, total_loss: 1.769652, forward_loss: 0.000000, inverse_pred_loss: 1.769362, Q_loss: 0.000290, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  312\n",
      "frames: 662000, reward: 17.700000, total_loss: 1.774779, forward_loss: 0.000000, inverse_pred_loss: 1.774635, Q_loss: 0.000144, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  313\n",
      "frames: 663000, reward: 17.700000, total_loss: 1.752538, forward_loss: 0.000000, inverse_pred_loss: 1.752427, Q_loss: 0.000110, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  313\n",
      "frames: 664000, reward: 18.500000, total_loss: 1.786862, forward_loss: 0.000000, inverse_pred_loss: 1.786740, Q_loss: 0.000122, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  314\n",
      "frames: 665000, reward: 18.400000, total_loss: 1.771747, forward_loss: 0.000000, inverse_pred_loss: 1.771566, Q_loss: 0.000181, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  315\n",
      "frames: 666000, reward: 18.400000, total_loss: 1.787590, forward_loss: 0.000000, inverse_pred_loss: 1.787434, Q_loss: 0.000156, intrinsic_rewards: 0.000000, epsilon: 0.010002, episode:  315\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-26-02a02841d3df>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     68\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     69\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmemory_buffer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>=\u001b[0m \u001b[0mlearning_start\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 70\u001b[1;33m         \u001b[0mloss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mQ_loss_record\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mforward_loss_record\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minverse_pred_loss_record\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mintrinsic_rewards_rec\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlearn_from_experience\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     71\u001b[0m         \u001b[0mlosses\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     72\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-24-3af40e6f8895>\u001b[0m in \u001b[0;36mlearn_from_experience\u001b[1;34m(self, batch_size)\u001b[0m\n\u001b[0;32m    117\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mlearn_from_experience\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    118\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmemory_buffer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 119\u001b[1;33m             \u001b[0mstates\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mactions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnext_states\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdones\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msample_from_buffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    120\u001b[0m             \u001b[0mtd_loss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mQ_loss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mforward_loss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minverse_pred_loss\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mintrinsic_rewards\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompute_td_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstates\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mactions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnext_states\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdones\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    121\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-24-3af40e6f8895>\u001b[0m in \u001b[0;36msample_from_buffer\u001b[1;34m(self, batch_size)\u001b[0m\n\u001b[0;32m    111\u001b[0m             \u001b[0mactions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    112\u001b[0m             \u001b[0mrewards\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreward\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 113\u001b[1;33m             \u001b[0mnext_states\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobserve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnext_frame\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    114\u001b[0m             \u001b[0mdones\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    115\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mactions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnext_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdones\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-24-3af40e6f8895>\u001b[0m in \u001b[0;36mobserve\u001b[1;34m(self, lazyframe)\u001b[0m\n\u001b[0;32m     28\u001b[0m         \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m  \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlazyframe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_force\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m/\u001b[0m\u001b[1;36m255\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mUSE_CUDA\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 30\u001b[1;33m             \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstate\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     31\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mstate\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     32\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# if __name__ == '__main__':\n",
    "    \n",
    "# Training DQN in PongNoFrameskip-v4 \n",
    "env = make_atari('PongNoFrameskip-v4')\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True)\n",
    "\n",
    "gamma = 0.99\n",
    "epsilon_max = 1\n",
    "epsilon_min = 0.01\n",
    "eps_decay = 50000\n",
    "frames = 1000000\n",
    "USE_CUDA = True\n",
    "learning_rate = 1e-4\n",
    "max_buff = 100000\n",
    "update_tar_interval = 1000\n",
    "batch_size = 32\n",
    "print_interval = 1000\n",
    "log_interval = 1000\n",
    "learning_start = 5000 # 10000\n",
    "win_reward = 18     # Pong-v4\n",
    "win_break = True\n",
    "\n",
    "# param for ICM\n",
    "forward_scale = 1 # scale for loss function of forward prediction model, 0.8        \n",
    "inverse_scale = 1 # scale for loss function of inverse prediction model, 0.2\n",
    "Qloss_scale = 1 # scale for loss function of Q value, 1\n",
    "intrinsic_scale = 100 # scale for intrinsic reward, 1\n",
    "use_extrinsic = True # whether use extrinsic rewards, if False, only intrinsic reward generated from ICM is used\n",
    "\n",
    "\n",
    "action_space = env.action_space\n",
    "action_dim = env.action_space.n\n",
    "state_dim = env.observation_space.shape[0]\n",
    "state_channel = env.observation_space.shape[2]\n",
    "agent = ICM_DQNAgent(in_channels = state_channel, action_space= action_space, USE_CUDA = USE_CUDA, lr = learning_rate,\n",
    "                    forward_scale = forward_scale, inverse_scale =inverse_scale, Qloss_scale = Qloss_scale, intrinsic_scale= intrinsic_scale,\n",
    "                     use_extrinsic = use_extrinsic)\n",
    "\n",
    "frame = env.reset()\n",
    "\n",
    "episode_reward = 0\n",
    "all_rewards = []\n",
    "losses = []\n",
    "episode_num = 0\n",
    "is_win = False\n",
    "# tensorboard\n",
    "summary_writer = SummaryWriter(log_dir = \"ICM_DQN3_Pong\", comment= \"good_makeatari\")\n",
    "\n",
    "# e-greedy decay\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_min + (epsilon_max - epsilon_min) * math.exp(\n",
    "            -1. * frame_idx / eps_decay)\n",
    "# plt.plot([epsilon_by_frame(i) for i in range(10000)])\n",
    "loss = 0\n",
    "Q_loss_record = 0\n",
    "forward_loss_record=0\n",
    "inverse_pred_loss_record=0\n",
    "intrinsic_rewards_rec = 0\n",
    "for i in range(frames):\n",
    "    epsilon = epsilon_by_frame(i)\n",
    "    state_tensor = agent.observe(frame)\n",
    "    action = agent.act(state_tensor, epsilon)\n",
    "    \n",
    "    next_frame, reward, done, _ = env.step(action)\n",
    "    \n",
    "    episode_reward += reward\n",
    "    agent.memory_buffer.push(frame, action, reward, next_frame, done)\n",
    "    frame = next_frame\n",
    "    \n",
    "    if agent.memory_buffer.size() >= learning_start:\n",
    "        loss, Q_loss_record, forward_loss_record, inverse_pred_loss_record, intrinsic_rewards_rec = agent.learn_from_experience(batch_size)\n",
    "        losses.append(loss)\n",
    "\n",
    "    if i % print_interval == 0:\n",
    "        print(\"frames: %5d, reward: %5f, total_loss: %4f, forward_loss: %4f, inverse_pred_loss: %4f, Q_loss: %4f, intrinsic_rewards: %4f, epsilon: %5f, episode: %4d\" % \n",
    "              (i, np.mean(all_rewards[-10:]), loss, forward_loss_record, inverse_pred_loss_record, Q_loss_record, intrinsic_rewards_rec, epsilon, episode_num))\n",
    "        summary_writer.add_scalar(\"Temporal Difference Loss\", loss, i)\n",
    "        summary_writer.add_scalar(\"Mean Reward\", np.mean(all_rewards[-10:]), i)\n",
    "        summary_writer.add_scalar(\"Epsilon\", epsilon, i)\n",
    "        \n",
    "    if i % update_tar_interval == 0:\n",
    "        agent.DQN_target.load_state_dict(agent.DQN.state_dict())\n",
    "    \n",
    "    if done:\n",
    "        \n",
    "        frame = env.reset()\n",
    "        \n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        episode_num += 1\n",
    "        avg_reward = float(np.mean(all_rewards[-100:]))\n",
    "\n",
    "summary_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_training(frame_idx, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    plt.show()\n",
    "\n",
    "plot_training(i, all_rewards, losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "?F.cross_entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
